From eb6aedf92c0fe467fd4724623262907ad78573bb Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Nov 2014 12:21:00 +0000 Subject: More work on pushers. Attempt to do HTTP pokes. Not sure if the actual HTTP pokes work or not yet but the retry semantics are pretty good. --- synapse/http/client.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'synapse/http') diff --git a/synapse/http/client.py b/synapse/http/client.py index 048a428905..82e80385ce 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -60,6 +60,25 @@ class SimpleHttpClient(object): defer.returnValue(json.loads(body)) + @defer.inlineCallbacks + def post_json_get_json(self, uri, post_json): + json_str = json.dumps(post_json) + + logger.info("HTTP POST %s -> %s", json_str, uri) + + response = yield self.agent.request( + "POST", + uri.encode("ascii"), + headers=Headers({ + "Content-Type": ["application/json"] + }), + bodyProducer=FileBodyProducer(StringIO(json_str)) + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + @defer.inlineCallbacks def get_json(self, uri, args={}): """ Get's some json from the given host and path -- cgit 1.5.1 From ede491b4e0c14d44ce43dd5b152abf148b54b9ed Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 17:38:53 +0000 Subject: Oops: second part of commit dc938606 --- synapse/api/errors.py | 12 ++++++++++++ synapse/http/server.py | 8 ++------ synapse/rest/__init__.py | 3 ++- synapse/storage/__init__.py | 3 +++ synapse/storage/schema/delta/v10.sql | 13 +++++++++++++ synapse/storage/schema/pusher.sql | 13 +++++++++++++ 6 files changed, 45 insertions(+), 7 deletions(-) (limited to 'synapse/http') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index a4155aebae..55181fe77e 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) class Codes(object): + UNRECOGNIZED = "M_UNRECOGNIZED" UNAUTHORIZED = "M_UNAUTHORIZED" FORBIDDEN = "M_FORBIDDEN" BAD_JSON = "M_BAD_JSON" @@ -82,6 +83,17 @@ class RegistrationError(SynapseError): pass +class UnrecognizedRequestError(SynapseError): + """An error indicating we don't understand the request you're trying to make""" + def __init__(self, *args, **kwargs): + if "errcode" not in kwargs: + kwargs["errcode"] = Codes.NOT_FOUND + super(UnrecognizedRequestError, self).__init__( + 400, + "Unrecognized request", + **kwargs + ) + class AuthError(SynapseError): """An error raised when there was a problem authorising an event.""" diff --git a/synapse/http/server.py b/synapse/http/server.py index 8015a22edf..0f6539e1be 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -16,7 +16,7 @@ from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException + cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError ) from synapse.util.logcontext import LoggingContext @@ -139,11 +139,7 @@ class JsonResource(HttpServer, resource.Resource): return # Huh. No one wanted to handle that? Fiiiiiine. Send 400. - self._send_response( - request, - 400, - {"error": "Unrecognized request"} - ) + raise UnrecognizedRequestError() except CodeMessageException as e: if isinstance(e, SynapseError): logger.info("%s SynapseError: %s - %s", request, e.code, e.msg) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 59521d0c77..8e5877cf3f 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -16,7 +16,7 @@ from . import ( room, events, register, login, profile, presence, initial_sync, directory, - voip, admin, pusher, + voip, admin, pusher, push_rule ) @@ -46,3 +46,4 @@ class RestServletFactory(object): voip.register_servlets(hs, client_resource) admin.register_servlets(hs, client_resource) pusher.register_servlets(hs, client_resource) + push_rule.register_servlets(hs, client_resource) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 191fe462a5..11706676d0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,6 +30,7 @@ from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore from .pusher import PusherStore +from .push_rule import PushRuleStore from .media_repository import MediaRepositoryStore from .state import StateStore @@ -62,6 +63,7 @@ SCHEMAS = [ "event_edges", "event_signatures", "pusher", + "push_rules", "media_repository", ] @@ -85,6 +87,7 @@ class DataStore(RoomMemberStore, RoomStore, EventFederationStore, MediaRepositoryStore, PusherStore, + PushRuleStore ): def __init__(self, hs): diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index b84ce20ef3..8c4dfd5c1b 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -31,3 +31,16 @@ CREATE TABLE IF NOT EXISTS pushers ( FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); + +CREATE TABLE IF NOT EXISTS push_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index b84ce20ef3..8c4dfd5c1b 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -31,3 +31,16 @@ CREATE TABLE IF NOT EXISTS pushers ( FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); + +CREATE TABLE IF NOT EXISTS push_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); -- cgit 1.5.1 From e0bf18addf643bd72a0cd5cb8dbc127dc7a7e8d2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 14:16:28 +0000 Subject: Add RestServlet base class in synapse/http/servlet.py --- synapse/http/servlet.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 synapse/http/servlet.py (limited to 'synapse/http') diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py new file mode 100644 index 0000000000..d5ccf2742f --- /dev/null +++ b/synapse/http/servlet.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains base REST classes for constructing REST servlets. """ + +import logging + + +logger = logging.getLogger(__name__) + + +class RestServlet(object): + + """ A Synapse REST Servlet. + + An implementing class can either provide its own custom 'register' method, + or use the automatic pattern handling provided by the base class. + + To use this latter, the implementing class instead provides a `PATTERN` + class attribute containing a pre-compiled regular expression. The automatic + register method will then use this method to register any of the following + instance methods associated with the corresponding HTTP method: + + on_GET + on_PUT + on_POST + on_DELETE + on_OPTIONS + + Automatically handles turning CodeMessageExceptions thrown by these methods + into the appropriate HTTP response. + """ + + def register(self, http_server): + """ Register this servlet with the given HTTP server. """ + if hasattr(self, "PATTERN"): + pattern = self.PATTERN + + for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"): + if hasattr(self, "on_%s" % (method)): + method_handler = getattr(self, "on_%s" % (method)) + http_server.register_path(method, pattern, method_handler) + else: + raise NotImplementedError("RestServlet must register something.") -- cgit 1.5.1 From 7b814d3f7fc8137426bc97fb80751753eb8eb94b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 18:54:51 +0000 Subject: Add client v2_alpha resource to synapse server resource tree --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 7 +++- synapse/http/servlet.py | 57 ++++++++++++++++++++++++++++++++ synapse/rest/client/v2_alpha/__init__.py | 29 ++++++++++++++++ synapse/rest/client/v2_alpha/_base.py | 38 +++++++++++++++++++++ synapse/server.py | 1 + 6 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/__init__.py create mode 100644 synapse/rest/client/v2_alpha/_base.py (limited to 'synapse/http') diff --git a/synapse/api/urls.py b/synapse/api/urls.py index a299392049..693c0efda6 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -16,6 +16,7 @@ """Contains the URL paths to prefix various aspects of the server with. """ CLIENT_PREFIX = "/_matrix/client/api/v1" +CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha" FEDERATION_PREFIX = "/_matrix/federation/v1" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fabe8ddacb..40d28dcbdc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -32,12 +32,13 @@ from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory from synapse.util.logcontext import LoggingContext from synapse.rest.client.v1 import ClientV1RestResource +from synapse.rest.client.v2_alpha import ClientV2AlphaRestResource from daemonize import Daemonize import twisted.manhole.telnet @@ -62,6 +63,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_client(self): return ClientV1RestResource(self) + def build_resource_for_client_v2_alpha(self): + return ClientV2AlphaRestResource(self) + def build_resource_for_federation(self): return JsonResource() @@ -105,6 +109,7 @@ class SynapseHomeServer(HomeServer): # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] desired_tree = [ (CLIENT_PREFIX, self.get_resource_for_client()), + (CLIENT_V2_ALPHA_PREFIX, self.get_resource_for_client_v2_alpha()), (FEDERATION_PREFIX, self.get_resource_for_federation()), (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index d5ccf2742f..a4eb6c817c 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -15,6 +15,8 @@ """ This module contains base REST classes for constructing REST servlets. """ +from synapse.api.errors import SynapseError + import logging @@ -54,3 +56,58 @@ class RestServlet(object): http_server.register_path(method, pattern, method_handler) else: raise NotImplementedError("RestServlet must register something.") + + @staticmethod + def parse_integer(request, name, default=None, required=False): + if name in request.args: + try: + return int(request.args[name][0]) + except: + message = "Query parameter %r must be an integer" % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing integer query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_boolean(request, name, default=None, required=False): + if name in request.args: + try: + return { + "true": True, + "false": False, + }[request.args[name][0]] + except: + message = ( + "Boolean query parameter %r must be one of" + " ['true', 'false']" + ) % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing boolean query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_string(request, name, default=None, required=False, + allowed_values=None, param_type="string"): + if name in request.args: + value = request.args[name][0] + if allowed_values is not None and value not in allowed_values: + message = "Query parameter %r must be one of [%s]" % ( + name, ", ".join(repr(v) for v in allowed_values) + ) + raise SynapseError(message) + else: + return value + else: + if required: + message = "Missing %s query parameter %r" % (param_type, name) + raise SynapseError(400, message) + else: + return default diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py new file mode 100644 index 0000000000..bb740e2803 --- /dev/null +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from synapse.http.server import JsonResource + + +class ClientV2AlphaRestResource(JsonResource): + """A resource for version 2 alpha of the matrix client API.""" + + def __init__(self, hs): + JsonResource.__init__(self) + self.register_servlets(self, hs) + + @staticmethod + def register_servlets(client_resource, hs): + pass diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py new file mode 100644 index 0000000000..22dc5cb862 --- /dev/null +++ b/synapse/rest/client/v2_alpha/_base.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def client_v2_pattern(path_regex): + """Creates a regex compiled client path with the correct client path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) diff --git a/synapse/server.py b/synapse/server.py index 32013b1a91..92ed2c5e32 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -70,6 +70,7 @@ class BaseHomeServer(object): 'notifier', 'distributor', 'resource_for_client', + 'resource_for_client_v2_alpha', 'resource_for_federation', 'resource_for_web_client', 'resource_for_content_repo', -- cgit 1.5.1 From c183cec8f61f5b3488973f01ca5203183a00e6d1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 13:44:52 +0000 Subject: Add post_json(...) method to federation client --- synapse/http/matrixfederationclient.py | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1dda3ba2c7..b1b2916fd3 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -244,6 +244,43 @@ class MatrixFederationHttpClient(object): defer.returnValue((response.code, body)) + @defer.inlineCallbacks + def post_json(self, destination, path, data={}): + """ Sends the specifed json data using POST + + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + data (dict): A dict containing the data that will be used as + the request body. This will be encoded as JSON. + + Returns: + Deferred: Succeeds when we get a 2xx HTTP response. The result + will be the decoded JSON body. On a 4xx or 5xx error response a + CodeMessageException is raised. + """ + + def body_callback(method, url_bytes, headers_dict): + self.sign_request( + destination, method, url_bytes, headers_dict, data + ) + return None + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + body_callback=body_callback, + headers_dict={"Content-Type": ["application/json"]}, + ) + + logger.debug("Getting resp body") + body = yield readBody(response) + logger.debug("Got resp body") + + defer.returnValue((response.code, body)) + @defer.inlineCallbacks def get_json(self, destination, path, args={}, retry_on_dns_fail=True): """ GETs some json from the given host homeserver and path -- cgit 1.5.1 From 5a3a15f5c186af6b818f2b2d3a4dafeee48b4e33 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 13:58:22 +0000 Subject: Make post_json(...) actually send data. --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b1b2916fd3..c7bf1b47b8 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -265,7 +265,7 @@ class MatrixFederationHttpClient(object): self.sign_request( destination, method, url_bytes, headers_dict, data ) - return None + return _JsonProducer(data) response = yield self._create_request( destination.encode("ascii"), -- cgit 1.5.1 From ae46f10fc5dba0f81518e3144ab8d9ed7a7d03bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 16:28:12 +0000 Subject: Apply sanity to the transport client interface. Convert 'make_join' and 'send_join' to accept iterables of destinations --- synapse/api/errors.py | 8 +++- synapse/federation/federation_client.py | 82 ++++++++++++++++++++------------- synapse/federation/transaction_queue.py | 23 +++++++-- synapse/federation/transport/client.py | 42 +++++++---------- synapse/handlers/federation.py | 4 +- synapse/http/matrixfederationclient.py | 42 ++++++++++++++--- 6 files changed, 130 insertions(+), 71 deletions(-) (limited to 'synapse/http') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index ad478aa6b7..5041828f18 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -39,7 +39,7 @@ class Codes(object): TOO_LARGE = "M_TOO_LARGE" -class CodeMessageException(Exception): +class CodeMessageException(RuntimeError): """An exception with integer code and message string attributes.""" def __init__(self, code, msg): @@ -227,3 +227,9 @@ class FederationError(RuntimeError): "affected": self.affected, "source": self.source if self.source else self.affected, } + + +class HttpResponseException(CodeMessageException): + def __init__(self, code, msg, response): + self.response = response + super(HttpResponseException, self).__init__(code, msg) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index d6b8c43916..eb36ec040b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -19,6 +19,7 @@ from twisted.internet import defer from .federation_base import FederationBase from .units import Edu +from synapse.api.errors import CodeMessageException from synapse.util.logutils import log_function from synapse.events import FrozenEvent @@ -180,7 +181,8 @@ class FederationClient(FederationBase): pdu = yield self._check_sigs_and_hash(pdu) break - + except CodeMessageException: + raise except Exception as e: logger.info( "Failed to get PDU %s from %s because %s", @@ -264,45 +266,63 @@ class FederationClient(FederationBase): defer.returnValue(self.event_from_pdu_json(pdu_dict)) break - except Exception as e: - logger.warn("Failed to make_join via %s", destination) + except CodeMessageException: + raise + except RuntimeError as e: + logger.warn( + "Failed to make_join via %s: %s", + destination, e.message + ) + + raise RuntimeError("Failed to send to any server.") @defer.inlineCallbacks - def send_join(self, destination, pdu): - time_now = self._clock.time_msec() - _, content = yield self.transport_layer.send_join( - destination=destination, - room_id=pdu.room_id, - event_id=pdu.event_id, - content=pdu.get_pdu_json(time_now), - ) + def send_join(self, destinations, pdu): + for destination in destinations: + try: + time_now = self._clock.time_msec() + _, content = yield self.transport_layer.send_join( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=pdu.get_pdu_json(time_now), + ) - logger.debug("Got content: %s", content) + logger.debug("Got content: %s", content) - state = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("state", []) - ] + state = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("state", []) + ] - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("auth_chain", []) - ] + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("auth_chain", []) + ] - signed_state = yield self._check_sigs_and_hash_and_fetch( - destination, state, outlier=True - ) + signed_state = yield self._check_sigs_and_hash_and_fetch( + destination, state, outlier=True + ) - signed_auth = yield self._check_sigs_and_hash_and_fetch( - destination, auth_chain, outlier=True - ) + signed_auth = yield self._check_sigs_and_hash_and_fetch( + destination, auth_chain, outlier=True + ) - auth_chain.sort(key=lambda e: e.depth) + auth_chain.sort(key=lambda e: e.depth) + + defer.returnValue({ + "state": signed_state, + "auth_chain": signed_auth, + }) + except CodeMessageException: + raise + except RuntimeError as e: + logger.warn( + "Failed to send_join via %s: %s", + destination, e.message + ) - defer.returnValue({ - "state": signed_state, - "auth_chain": signed_auth, - }) + raise RuntimeError("Failed to send to any server.") @defer.inlineCallbacks def send_invite(self, destination, room_id, event_id, pdu): diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 9d4f2c09a2..f38aeba7cc 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -19,6 +19,7 @@ from twisted.internet import defer from .persistence import TransactionActions from .units import Transaction +from synapse.api.errors import HttpResponseException from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext @@ -238,9 +239,14 @@ class TransactionQueue(object): del p["age_ts"] return data - code, response = yield self.transport_layer.send_transaction( - transaction, json_data_cb - ) + try: + response = yield self.transport_layer.send_transaction( + transaction, json_data_cb + ) + code = 200 + except HttpResponseException as e: + code = e.code + response = e.response logger.info("TX [%s] got %d response", destination, code) @@ -274,8 +280,7 @@ class TransactionQueue(object): pass logger.debug("TX [%s] Yielded to callbacks", destination) - - except Exception as e: + except RuntimeError as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. logger.warn( @@ -283,6 +288,14 @@ class TransactionQueue(object): destination, e, ) + except Exception as e: + # We capture this here as there as nothing actually listens + # for this finishing functions deferred. + logger.exception( + "TX [%s] Problem in _attempt_transaction: %s", + destination, + e, + ) self.set_retrying(destination, retry_interval) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 4cb1dea2de..8b137e7128 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -19,7 +19,6 @@ from synapse.api.urls import FEDERATION_PREFIX as PREFIX from synapse.util.logutils import log_function import logging -import json logger = logging.getLogger(__name__) @@ -129,7 +128,7 @@ class TransportLayerClient(object): # generated by the json_data_callback. json_data = transaction.get_dict() - code, response = yield self.client.put_json( + response = yield self.client.put_json( transaction.destination, path=PREFIX + "/send/%s/" % transaction.transaction_id, data=json_data, @@ -137,95 +136,86 @@ class TransportLayerClient(object): ) logger.debug( - "send_data dest=%s, txid=%s, got response: %d", - transaction.destination, transaction.transaction_id, code + "send_data dest=%s, txid=%s, got response: 200", + transaction.destination, transaction.transaction_id, ) - defer.returnValue((code, response)) + defer.returnValue(response) @defer.inlineCallbacks @log_function def make_query(self, destination, query_type, args, retry_on_dns_fail): path = PREFIX + "/query/%s" % query_type - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, args=args, retry_on_dns_fail=retry_on_dns_fail, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def make_join(self, destination, room_id, user_id, retry_on_dns_fail=True): path = PREFIX + "/make_join/%s/%s" % (room_id, user_id) - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, retry_on_dns_fail=retry_on_dns_fail, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def send_join(self, destination, room_id, event_id, content): path = PREFIX + "/send_join/%s/%s" % (room_id, event_id) - code, content = yield self.client.put_json( + response = yield self.client.put_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_join", code) - - defer.returnValue(json.loads(content)) + defer.returnValue(response) @defer.inlineCallbacks @log_function def send_invite(self, destination, room_id, event_id, content): path = PREFIX + "/invite/%s/%s" % (room_id, event_id) - code, content = yield self.client.put_json( + response = yield self.client.put_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_invite", code) - - defer.returnValue(json.loads(content)) + defer.returnValue(response) @defer.inlineCallbacks @log_function def get_event_auth(self, destination, room_id, event_id): path = PREFIX + "/event_auth/%s/%s" % (room_id, event_id) - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def send_query_auth(self, destination, room_id, event_id, content): path = PREFIX + "/query_auth/%s/%s" % (room_id, event_id) - code, content = yield self.client.post_json( + content = yield self.client.post_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_invite", code) - - defer.returnValue(json.loads(content)) + defer.returnValue(content) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0876589e31..a968a87360 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -288,7 +288,7 @@ class FederationHandler(BaseHandler): logger.debug("Joining %s to %s", joinee, room_id) pdu = yield self.replication_layer.make_join( - target_host, + [target_host], room_id, joinee ) @@ -331,7 +331,7 @@ class FederationHandler(BaseHandler): new_event = builder.build() ret = yield self.replication_layer.send_join( - target_host, + [target_host], new_event ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index c7bf1b47b8..8559d06b7f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -27,7 +27,9 @@ from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json -from synapse.api.errors import CodeMessageException, SynapseError, Codes +from synapse.api.errors import ( + SynapseError, Codes, HttpResponseException, +) from syutil.crypto.jsonsign import sign_json @@ -163,13 +165,12 @@ class MatrixFederationHttpClient(object): ) if 200 <= response.code < 300: - # We need to update the transactions table to say it was sent? pass else: # :'( # Update transactions table? - raise CodeMessageException( - response.code, response.phrase + raise HttpResponseException( + response.code, response.phrase, response ) defer.returnValue(response) @@ -238,11 +239,20 @@ class MatrixFederationHttpClient(object): headers_dict={"Content-Type": ["application/json"]}, ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + logger.debug("Getting resp body") body = yield readBody(response) logger.debug("Got resp body") - defer.returnValue((response.code, body)) + defer.returnValue(json.loads(body)) @defer.inlineCallbacks def post_json(self, destination, path, data={}): @@ -275,11 +285,20 @@ class MatrixFederationHttpClient(object): headers_dict={"Content-Type": ["application/json"]}, ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + logger.debug("Getting resp body") body = yield readBody(response) logger.debug("Got resp body") - defer.returnValue((response.code, body)) + defer.returnValue(json.loads(body)) @defer.inlineCallbacks def get_json(self, destination, path, args={}, retry_on_dns_fail=True): @@ -321,7 +340,18 @@ class MatrixFederationHttpClient(object): retry_on_dns_fail=retry_on_dns_fail ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + + logger.debug("Getting resp body") body = yield readBody(response) + logger.debug("Got resp body") defer.returnValue(json.loads(body)) -- cgit 1.5.1 From 96d4bf90120a07faa5163c2e5af542358554dd61 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:07:31 +0000 Subject: Modify API for SimpleHttpClient.get_json and update usages. Previously, this would only return the HTTP body as JSON, and discard other response information (e.g. the HTTP response code). This has now been changed to throw a CodeMessageException on a non-2xx response, with the response code and body, which can then be parsed as JSON. Affected modules include: - Registration/Login (when using an email for IS auth) --- synapse/handlers/login.py | 33 +++++++++++++++----------- synapse/handlers/register.py | 56 ++++++++++++++++++++++++++------------------ synapse/http/client.py | 19 +++++++-------- 3 files changed, 61 insertions(+), 47 deletions(-) (limited to 'synapse/http') diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index d297d71c03..7447800460 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -16,12 +16,13 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.errors import LoginError, Codes +from synapse.api.errors import LoginError, Codes, CodeMessageException from synapse.http.client import SimpleHttpClient from synapse.util.emailutils import EmailException import synapse.util.emailutils as emailutils import bcrypt +import json import logging logger = logging.getLogger(__name__) @@ -96,16 +97,20 @@ class LoginHandler(BaseHandler): @defer.inlineCallbacks def _query_email(self, email): - httpCli = SimpleHttpClient(self.hs) - data = yield httpCli.get_json( - # TODO FIXME This should be configurable. - # XXX: ID servers need to use HTTPS - "http://%s%s" % ( - "matrix.org:8090", "/_matrix/identity/api/v1/lookup" - ), - { - 'medium': 'email', - 'address': email - } - ) - defer.returnValue(data) + http_client = SimpleHttpClient(self.hs) + try: + data = yield http_client.get_json( + # TODO FIXME This should be configurable. + # XXX: ID servers need to use HTTPS + "http://%s%s" % ( + "matrix.org:8090", "/_matrix/identity/api/v1/lookup" + ), + { + 'medium': 'email', + 'address': email + } + ) + defer.returnValue(data) + except CodeMessageException as e: + data = json.loads(e.msg) + defer.returnValue(data) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 66a89c10b2..08cd5fd720 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.types import UserID from synapse.api.errors import ( - SynapseError, RegistrationError, InvalidCaptchaError + SynapseError, RegistrationError, InvalidCaptchaError, CodeMessageException ) from ._base import BaseHandler import synapse.util.stringutils as stringutils @@ -28,6 +28,7 @@ from synapse.http.client import CaptchaServerHttpClient import base64 import bcrypt +import json import logging logger = logging.getLogger(__name__) @@ -161,21 +162,26 @@ class RegistrationHandler(BaseHandler): def _threepid_from_creds(self, creds): # TODO: get this from the homeserver rather than creating a new one for # each request - httpCli = SimpleHttpClient(self.hs) + http_client = SimpleHttpClient(self.hs) # XXX: make this configurable! trustedIdServers = ['matrix.org:8090', 'matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) defer.returnValue(None) - data = yield httpCli.get_json( - # XXX: This should be HTTPS - "http://%s%s" % ( - creds['idServer'], - "/_matrix/identity/api/v1/3pid/getValidated3pid" - ), - {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} - ) + + data = {} + try: + data = yield http_client.get_json( + # XXX: This should be HTTPS + "http://%s%s" % ( + creds['idServer'], + "/_matrix/identity/api/v1/3pid/getValidated3pid" + ), + {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} + ) + except CodeMessageException as e: + data = json.loads(e.msg) if 'medium' in data: defer.returnValue(data) @@ -185,19 +191,23 @@ class RegistrationHandler(BaseHandler): def _bind_threepid(self, creds, mxid): yield logger.debug("binding threepid") - httpCli = SimpleHttpClient(self.hs) - data = yield httpCli.post_urlencoded_get_json( - # XXX: Change when ID servers are all HTTPS - "http://%s%s" % ( - creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" - ), - { - 'sid': creds['sid'], - 'clientSecret': creds['clientSecret'], - 'mxid': mxid, - } - ) - logger.debug("bound threepid") + http_client = SimpleHttpClient(self.hs) + data = None + try: + data = yield http_client.post_urlencoded_get_json( + # XXX: Change when ID servers are all HTTPS + "http://%s%s" % ( + creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" + ), + { + 'sid': creds['sid'], + 'clientSecret': creds['clientSecret'], + 'mxid': mxid, + } + ) + logger.debug("bound threepid") + except CodeMessageException as e: + data = json.loads(e.msg) defer.returnValue(data) @defer.inlineCallbacks diff --git a/synapse/http/client.py b/synapse/http/client.py index 198f575cfa..5f4558be47 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from synapse.api.errors import CodeMessageException from synapse.http.agent_name import AGENT_NAME from twisted.internet import defer, reactor from twisted.web.client import ( @@ -83,7 +83,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def get_json(self, uri, args={}): - """ Get's some json from the given host and path + """ Gets some json from the given host and path Args: uri (str): The URI to request, not including query parameters @@ -91,15 +91,11 @@ class SimpleHttpClient(object): None. **Note**: The value of each key is assumed to be an iterable and *not* a string. - Returns: - Deferred: Succeeds when we get *any* HTTP response. - - The result of the deferred is a tuple of `(code, response)`, - where `response` is a dict representing the decoded JSON body. + Deferred: Succeeds when we get *any* 2xx HTTP response. + Raises: + On a non-2xx HTTP response. """ - - yield if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) @@ -114,7 +110,10 @@ class SimpleHttpClient(object): body = yield readBody(response) - defer.returnValue(json.loads(body)) + if 200 <= response.code < 300: + defer.returnValue(json.loads(body)) + else: + raise CodeMessageException(response.code, body) class CaptchaServerHttpClient(SimpleHttpClient): -- cgit 1.5.1 From 6d3e4f4d0aad4ad9b44b28349838ff48aef39440 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:32:44 +0000 Subject: Update user/alias query APIs to use new format of SimpleHttpClient.get_json --- synapse/appservice/api.py | 15 +++++++-------- synapse/http/client.py | 3 +++ 2 files changed, 10 insertions(+), 8 deletions(-) (limited to 'synapse/http') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 799ada96df..74508ecddf 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from twisted.internet import defer -from twisted.web.client import PartialDownloadError +from synapse.api.errors import CodeMessageException from synapse.http.client import SimpleHttpClient import logging @@ -42,11 +42,11 @@ class ApplicationServiceApi(SimpleHttpClient): }) if response: # just an empty json object defer.returnValue(True) - except PartialDownloadError as e: - if e.status == 404: + except CodeMessageException as e: + if e.code == 404: defer.returnValue(False) return - logger.warning("query_user to %s received %s", (uri, e.status)) + logger.warning("query_user to %s received %s", uri, e.code) @defer.inlineCallbacks def query_alias(self, service, alias): @@ -56,14 +56,13 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": self.hs_token }) - logger.info("%s", response[0]) if response: # just an empty json object defer.returnValue(True) - except PartialDownloadError as e: - if e.status == 404: + except CodeMessageException as e: + if e.code == 404: defer.returnValue(False) return - logger.warning("query_alias to %s received %s", (uri, e.status)) + logger.warning("query_alias to %s received %s", uri, e.code) def push_bulk(self, service, events): pass diff --git a/synapse/http/client.py b/synapse/http/client.py index 5f4558be47..fee8c901a2 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -113,6 +113,9 @@ class SimpleHttpClient(object): if 200 <= response.code < 300: defer.returnValue(json.loads(body)) else: + # NB: This is explicitly not json.loads(body)'d because the contract + # of CodeMessageException is a *string* message. Callers can always + # load it into JSON if they want. raise CodeMessageException(response.code, body) -- cgit 1.5.1 From 543e84fe70bdc0af6be5feb237de2d18adf352ab Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:39:51 +0000 Subject: Add SimpleHttpClient.put_json with the same semantics as get_json. --- synapse/http/client.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/client.py b/synapse/http/client.py index fee8c901a2..575510637e 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -83,7 +83,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def get_json(self, uri, args={}): - """ Gets some json from the given host and path + """ Gets some json from the given URI. Args: uri (str): The URI to request, not including query parameters @@ -92,7 +92,8 @@ class SimpleHttpClient(object): **Note**: The value of each key is assumed to be an iterable and *not* a string. Returns: - Deferred: Succeeds when we get *any* 2xx HTTP response. + Deferred: Succeeds when we get *any* 2xx HTTP response, with the + HTTP body as JSON. Raises: On a non-2xx HTTP response. """ @@ -118,6 +119,49 @@ class SimpleHttpClient(object): # load it into JSON if they want. raise CodeMessageException(response.code, body) + @defer.inlineCallbacks + def put_json(self, uri, json_body, args={}): + """ Puts some json to the given URI. + + Args: + uri (str): The URI to request, not including query parameters + json_body (dict): The JSON to put in the HTTP body, + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + Returns: + Deferred: Succeeds when we get *any* 2xx HTTP response, with the + HTTP body as JSON. + Raises: + On a non-2xx HTTP response. + """ + if len(args): + query_bytes = urllib.urlencode(args, True) + uri = "%s?%s" % (uri, query_bytes) + + json_str = json.dumps(json_body) + + response = yield self.agent.request( + "PUT", + uri.encode("ascii"), + headers=Headers({ + b"User-Agent": [AGENT_NAME], + "Content-Type": ["application/json"] + }), + bodyProducer=FileBodyProducer(StringIO(json_str)) + ) + + body = yield readBody(response) + + if 200 <= response.code < 300: + defer.returnValue(json.loads(body)) + else: + # NB: This is explicitly not json.loads(body)'d because the contract + # of CodeMessageException is a *string* message. Callers can always + # load it into JSON if they want. + raise CodeMessageException(response.code, body) + class CaptchaServerHttpClient(SimpleHttpClient): """ -- cgit 1.5.1 From 9f2573eea1641806c67d51b98013a17a686c0909 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 10:55:01 +0000 Subject: Return body of response in HttpResponseException --- synapse/http/matrixfederationclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 8559d06b7f..056d446e42 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -169,8 +169,9 @@ class MatrixFederationHttpClient(object): else: # :'( # Update transactions table? + body = yield readBody(response) raise HttpResponseException( - response.code, response.phrase, response + response.code, response.phrase, body ) defer.returnValue(response) -- cgit 1.5.1 From 24cc6979fb384ef383309b27d06985ba3a845b2b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 13:46:22 +0000 Subject: Log when we receive a request, when we send a response and how long it took to process it. --- synapse/app/homeserver.py | 2 +- synapse/http/server.py | 23 +++++++++++++++++++++-- synapse/rest/client/v1/__init__.py | 2 +- synapse/rest/client/v2_alpha/__init__.py | 2 +- 4 files changed, 24 insertions(+), 5 deletions(-) (limited to 'synapse/http') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f20ccfb5b6..0f175ec3f4 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -67,7 +67,7 @@ class SynapseHomeServer(HomeServer): return ClientV2AlphaRestResource(self) def build_resource_for_federation(self): - return JsonResource() + return JsonResource(self) def build_resource_for_web_client(self): syweb_path = os.path.dirname(syweb.__file__) diff --git a/synapse/http/server.py b/synapse/http/server.py index 0f6539e1be..6d084fa33c 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -69,9 +69,10 @@ class JsonResource(HttpServer, resource.Resource): _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"]) - def __init__(self): + def __init__(self, hs): resource.Resource.__init__(self) + self.clock = hs.get_clock() self.path_regexs = {} def register_path(self, method, path_pattern, callback): @@ -111,6 +112,7 @@ class JsonResource(HttpServer, resource.Resource): This checks if anyone has registered a callback for that method and path. """ + code = None try: # Just say yes to OPTIONS. if request.method == "OPTIONS": @@ -130,6 +132,13 @@ class JsonResource(HttpServer, resource.Resource): urllib.unquote(u).decode("UTF-8") for u in m.groups() ] + logger.info( + "Received request: %s %s", + request.method, request.path + ) + + start = self.clock.time_msec() + code, response = yield path_entry.callback( request, *args @@ -145,9 +154,11 @@ class JsonResource(HttpServer, resource.Resource): logger.info("%s SynapseError: %s - %s", request, e.code, e.msg) else: logger.exception(e) + + code = e.code self._send_response( request, - e.code, + code, cs_exception(e), response_code_message=e.response_code_message ) @@ -158,6 +169,14 @@ class JsonResource(HttpServer, resource.Resource): 500, {"error": "Internal server error"} ) + finally: + code = str(code) if code else "-" + + end = self.clock.time_msec() + logger.info( + "Processed request: %dms %s %s %s", + end-start, code, request.method, request.path + ) def _send_response(self, request, code, response_json_object, response_code_message=None): diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index d8d01cdd16..21876b3487 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -25,7 +25,7 @@ class ClientV1RestResource(JsonResource): """A resource for version 1 of the matrix client API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 8f611de3a8..bca65f2a6a 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -25,7 +25,7 @@ class ClientV2AlphaRestResource(JsonResource): """A resource for version 2 alpha of the matrix client API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod -- cgit 1.5.1 From 0e6b3e4e40ae918eacdef87bb50ff1d19b304e7c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 18:17:27 +0000 Subject: Time out HTTP federation requests --- synapse/http/matrixfederationclient.py | 11 +++++++++-- synapse/util/async.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index aa14782b0f..fdc1e2678e 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -22,7 +22,7 @@ from twisted.web._newclient import ResponseDone from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint -from synapse.util.async import sleep +from synapse.util.async import sleep, time_bound_deferred from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json @@ -78,6 +78,7 @@ class MatrixFederationHttpClient(object): self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname self.agent = MatrixFederationHttpAgent(reactor) + self.clock = hs.get_clock() @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, @@ -117,7 +118,7 @@ class MatrixFederationHttpClient(object): try: with PreserveLoggingContext(): - response = yield self.agent.request( + request_deferred = self.agent.request( destination, endpoint, method, @@ -128,6 +129,12 @@ class MatrixFederationHttpClient(object): producer ) + response = yield time_bound_deferred( + request_deferred, + clock=self.clock, + time_out=60, + ) + logger.debug("Got response to %s", method) break except Exception as e: diff --git a/synapse/util/async.py b/synapse/util/async.py index c4fe5d522f..d4d1d4b472 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -32,3 +32,23 @@ def run_on_reactor(): iteration of the main loop """ return sleep(0) + + +def time_bound_deferred(given_deferred, clock, time_out): + ret_deferred = defer.Deferred() + + def timed_out(): + if not given_deferred.called: + given_deferred.cancel() + ret_deferred.errback(RuntimeError("Timed out")) + + timer = clock.call_later(time_out, timed_out) + + def succeed(result): + clock.cancel_call_later(timer) + ret_deferred.callback(result) + + given_deferred.addCallback(succeed) + given_deferred.addErrback(ret_deferred.errback) + + return ret_deferred -- cgit 1.5.1 From dcf52469e821b3b2b69a0610c2c4f025a5aac68f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 10:25:06 +0000 Subject: Move time_bound_deferred into Clock --- synapse/http/matrixfederationclient.py | 5 ++--- synapse/util/__init__.py | 21 ++++++++++++++++++++- synapse/util/async.py | 20 -------------------- 3 files changed, 22 insertions(+), 24 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index fdc1e2678e..74e523960f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -22,7 +22,7 @@ from twisted.web._newclient import ResponseDone from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint -from synapse.util.async import sleep, time_bound_deferred +from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json @@ -129,9 +129,8 @@ class MatrixFederationHttpClient(object): producer ) - response = yield time_bound_deferred( + response = yield self.clock.time_bound_deferred( request_deferred, - clock=self.clock, time_out=60, ) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 4e837a918e..2da8dfa719 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -15,7 +15,7 @@ from synapse.util.logcontext import LoggingContext -from twisted.internet import reactor +from twisted.internet import defer, reactor import time @@ -45,3 +45,22 @@ class Clock(object): def cancel_call_later(self, timer): timer.cancel() + + def time_bound_deferred(self, given_deferred, time_out): + ret_deferred = defer.Deferred() + + def timed_out(): + if not given_deferred.called: + given_deferred.cancel() + ret_deferred.errback(RuntimeError("Timed out")) + + timer = self.call_later(time_out, timed_out) + + def succeed(result): + self.cancel_call_later(timer) + ret_deferred.callback(result) + + given_deferred.addCallback(succeed) + given_deferred.addErrback(ret_deferred.errback) + + return ret_deferred diff --git a/synapse/util/async.py b/synapse/util/async.py index d4d1d4b472..c4fe5d522f 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -32,23 +32,3 @@ def run_on_reactor(): iteration of the main loop """ return sleep(0) - - -def time_bound_deferred(given_deferred, clock, time_out): - ret_deferred = defer.Deferred() - - def timed_out(): - if not given_deferred.called: - given_deferred.cancel() - ret_deferred.errback(RuntimeError("Timed out")) - - timer = clock.call_later(time_out, timed_out) - - def succeed(result): - clock.cancel_call_later(timer) - ret_deferred.callback(result) - - given_deferred.addCallback(succeed) - given_deferred.addErrback(ret_deferred.errback) - - return ret_deferred -- cgit 1.5.1 From 05b961d7e3f6c926520c3200ae3031c77cebc212 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 10:28:46 +0000 Subject: PEP8 --- synapse/http/matrixfederationclient.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 74e523960f..92f887f778 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -72,7 +72,6 @@ class MatrixFederationHttpClient(object): requests. """ - def __init__(self, hs): self.hs = hs self.signing_key = hs.config.signing_key[0] -- cgit 1.5.1 From 6370cffbbfc0bcfa4465453c24c535ef755db4f1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 10:34:41 +0000 Subject: Fix bug where variable was not always defined --- synapse/http/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/server.py b/synapse/http/server.py index 6d084fa33c..589c30c199 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -113,6 +113,7 @@ class JsonResource(HttpServer, resource.Resource): path. """ code = None + start = self.clock.time_msec() try: # Just say yes to OPTIONS. if request.method == "OPTIONS": @@ -137,8 +138,6 @@ class JsonResource(HttpServer, resource.Resource): request.method, request.path ) - start = self.clock.time_msec() - code, response = yield path_entry.callback( request, *args -- cgit 1.5.1 From 4ebbaf0d4382813ba896f3e8101de12e112cbed5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 14:23:10 +0000 Subject: Blunty replace json with simplejson --- synapse/crypto/keyclient.py | 2 +- synapse/federation/persistence.py | 2 +- synapse/federation/transport/server.py | 2 +- synapse/http/client.py | 2 +- synapse/http/matrixfederationclient.py | 2 +- synapse/push/__init__.py | 2 +- synapse/push/pusherpool.py | 2 +- synapse/rest/client/v1/directory.py | 2 +- synapse/rest/client/v1/login.py | 2 +- synapse/rest/client/v1/presence.py | 2 +- synapse/rest/client/v1/profile.py | 2 +- synapse/rest/client/v1/push_rule.py | 2 +- synapse/rest/client/v1/pusher.py | 2 +- synapse/rest/client/v1/register.py | 2 +- synapse/rest/client/v1/room.py | 2 +- synapse/rest/client/v2_alpha/filter.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/storage/__init__.py | 2 +- synapse/storage/_base.py | 2 +- synapse/storage/filtering.py | 2 +- synapse/storage/push_rule.py | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) (limited to 'synapse/http') diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index cd12349f67..74008347c3 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -19,7 +19,7 @@ from twisted.internet.protocol import Factory from twisted.internet import defer, reactor from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.logcontext import PreserveLoggingContext -import json +import simplejson as json import logging diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 85c82a4623..8a1afc0ca5 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -23,7 +23,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function -import json +import simplejson as json import logging diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 9c9f8d525b..2ffb37aa18 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -20,7 +20,7 @@ from synapse.api.errors import Codes, SynapseError from synapse.util.logutils import log_function import logging -import json +import simplejson as json import re diff --git a/synapse/http/client.py b/synapse/http/client.py index 198f575cfa..d500e19c81 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -23,7 +23,7 @@ from twisted.web.http_headers import Headers from StringIO import StringIO -import json +import simplejson as json import logging import urllib diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 056d446e42..406203acf2 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -33,7 +33,7 @@ from synapse.api.errors import ( from syutil.crypto.jsonsign import sign_json -import json +import simplejson as json import logging import urllib import urlparse diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 418a348a58..0659a1cb9b 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -22,7 +22,7 @@ import synapse.util.async import baserules import logging -import json +import simplejson as json import re logger = logging.getLogger(__name__) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 5a525befd7..7483d257bf 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -20,7 +20,7 @@ from httppusher import HttpPusher from synapse.push import PusherConfigException import logging -import json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 8ed7e2d669..420aa89f38 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -20,7 +20,7 @@ from synapse.api.errors import AuthError, SynapseError, Codes from synapse.types import RoomAlias from .base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json import logging diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 7116ac98e8..b2257b749d 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError from synapse.types import UserID from base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json class LoginRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 7feb4aadb1..78d4f2b128 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -21,7 +21,7 @@ from synapse.api.errors import SynapseError from synapse.types import UserID from .base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json import logging logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 15d6f3fc6c..1e77eb49cf 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -19,7 +19,7 @@ from twisted.internet import defer from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID -import json +import simplejson as json class ProfileDisplaynameRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index c4e7dfcf0e..b012f31084 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -27,7 +27,7 @@ from synapse.push.rulekinds import ( PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP ) -import json +import simplejson as json class PushRuleRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 80e9939b79..6045e86f34 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError, Codes from synapse.push import PusherConfigException from .base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json class PusherRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index c0423c2d45..d3399c446b 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -25,7 +25,7 @@ from synapse.util.async import run_on_reactor from hashlib import sha1 import hmac -import json +import simplejson as json import logging import urllib diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 410f19ccf6..0346afb1b4 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -23,7 +23,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID, RoomAlias from synapse.events.utils import serialize_event -import json +import simplejson as json import logging import urllib diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 6ddc495d23..703250cea8 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -21,7 +21,7 @@ from synapse.types import UserID from ._base import client_v2_pattern -import json +import simplejson as json import logging diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index 22e26e3cd5..e77a20fb2e 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -25,7 +25,7 @@ from twisted.web import server, resource from twisted.internet import defer import base64 -import json +import simplejson as json import logging import os import re diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a63c59a8a2..924ea89035 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -44,7 +44,7 @@ from syutil.jsonutil import encode_canonical_json from synapse.crypto.event_signing import compute_event_reference_hash -import json +import simplejson as json import logging import os diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 3e1ab0a159..b74e74ac91 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -23,7 +23,7 @@ from synapse.util.logcontext import PreserveLoggingContext, LoggingContext from twisted.internet import defer import collections -import json +import simplejson as json import sys import time diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index e86eeced45..457a11fd02 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import SQLBaseStore -import json +import simplejson as json class FilteringStore(SQLBaseStore): diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 620de71398..ae46b39cc1 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -20,7 +20,7 @@ from twisted.internet import defer import logging import copy -import json +import simplejson as json logger = logging.getLogger(__name__) -- cgit 1.5.1 From f51832442674a1f72c41ffce2279880109fc7ff0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 16:41:16 +0000 Subject: Minor tweaks based on PR feedback. --- synapse/appservice/api.py | 6 +++--- synapse/http/client.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'synapse/http') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 6192813c03..c2179f8d55 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -80,11 +80,11 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.put_json( - uri, - { + uri=uri, + json_body={ "events": events }, - { + args={ "access_token": service.hs_token }) if response: # just an empty json object diff --git a/synapse/http/client.py b/synapse/http/client.py index 575510637e..7b23116556 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -95,7 +95,8 @@ class SimpleHttpClient(object): Deferred: Succeeds when we get *any* 2xx HTTP response, with the HTTP body as JSON. Raises: - On a non-2xx HTTP response. + On a non-2xx HTTP response. The response body will be used as the + error message. """ if len(args): query_bytes = urllib.urlencode(args, True) -- cgit 1.5.1 From 2bf0e85f3d43d41f450df4c475c9deb542999d82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 17:34:23 +0000 Subject: Use encode_canonical_json for http client --- synapse/http/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/client.py b/synapse/http/client.py index d500e19c81..43b2329780 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -15,6 +15,8 @@ from synapse.http.agent_name import AGENT_NAME +from syutil.jsonutil import encode_canonical_json + from twisted.internet import defer, reactor from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError @@ -64,7 +66,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def post_json_get_json(self, uri, post_json): - json_str = json.dumps(post_json) + json_str = encode_canonical_json(post_json) logger.info("HTTP POST %s -> %s", json_str, uri) -- cgit 1.5.1 From 4de93001bf6a7d8e770b990ea3546237b2569609 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:12:06 +0000 Subject: Make matrixfederationclient log more nicely --- synapse/http/matrixfederationclient.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1927948001..764b151d9a 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -16,6 +16,7 @@ from twisted.internet import defer, reactor, protocol from twisted.internet.error import DNSLookupError +from twisted.python.failure import Failure from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone @@ -146,14 +147,22 @@ class MatrixFederationHttpClient(object): ) raise SynapseError(400, "Domain specified not found.") + if hasattr(e, "reasons"): + reasons = ", ".join( + f.value.message + for f in e.reasons + ) + else: + reasons = e.message + logger.warn( - "Sending request failed to %s: %s %s : %s", + "Sending request failed to %s: %s %s: %s - %s", destination, method, url_bytes, - e + type(e). __name__, + reasons, ) - _print_ex(e) if retries_left: yield sleep(2 ** (5 - retries_left)) @@ -447,14 +456,6 @@ def _readBodyToFile(response, stream, max_size): return d -def _print_ex(e): - if hasattr(e, "reasons") and e.reasons: - for ex in e.reasons: - _print_ex(ex) - else: - logger.warn(e) - - class _JsonProducer(object): """ Used by the twisted http client to create the HTTP body from json """ -- cgit 1.5.1 From 0647e27a414e5a86cab57bba65931376e855c289 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:19:54 +0000 Subject: Remove unused import --- synapse/http/matrixfederationclient.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 764b151d9a..454c3d4ab1 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -16,7 +16,6 @@ from twisted.internet import defer, reactor, protocol from twisted.internet.error import DNSLookupError -from twisted.python.failure import Failure from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone -- cgit 1.5.1 From 0db52d43fa41526b83d0c70141c43bfd89388820 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:41:46 +0000 Subject: strings.join() expects iterable of strings --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 454c3d4ab1..b5e201b5f1 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -148,7 +148,7 @@ class MatrixFederationHttpClient(object): if hasattr(e, "reasons"): reasons = ", ".join( - f.value.message + str(f.value.message) for f in e.reasons ) else: -- cgit 1.5.1 From 5e2447146982a30d0d4ea1334297cc0bfb32c3c5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:50:10 +0000 Subject: Fix up ResponseNeverReceived to str --- synapse/http/matrixfederationclient.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b5e201b5f1..304efcdd56 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -146,21 +146,13 @@ class MatrixFederationHttpClient(object): ) raise SynapseError(400, "Domain specified not found.") - if hasattr(e, "reasons"): - reasons = ", ".join( - str(f.value.message) - for f in e.reasons - ) - else: - reasons = e.message - logger.warn( "Sending request failed to %s: %s %s: %s - %s", destination, method, url_bytes, type(e). __name__, - reasons, + _flatten_response_never_received(e), ) if retries_left: @@ -474,3 +466,13 @@ class _JsonProducer(object): def stopProducing(self): pass + + +def _flatten_response_never_received(e): + if hasattr(e, "reasons"): + return ", ".join( + _flatten_response_never_received(f.value) + for f in e.reasons + ) + else: + return "%s: %s" % (type(e), e.message,) -- cgit 1.5.1 From 65ca713ff53794bd517a85de7df063e631fb255a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:51:32 +0000 Subject: Add .__name__ after type(e) --- synapse/http/matrixfederationclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 304efcdd56..49a4c7d9d3 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -151,7 +151,7 @@ class MatrixFederationHttpClient(object): destination, method, url_bytes, - type(e). __name__, + type(e).__name__, _flatten_response_never_received(e), ) @@ -475,4 +475,4 @@ def _flatten_response_never_received(e): for f in e.reasons ) else: - return "%s: %s" % (type(e), e.message,) + return "%s: %s" % (type(e).__name__, e.message,) -- cgit 1.5.1 From 7e9d59f3b4628981d5e53ac0cc71325ad896287c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:58:13 +0000 Subject: Don't convert DNSLookupError to a 4xx SynapseError --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 49a4c7d9d3..fe57363388 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -144,7 +144,7 @@ class MatrixFederationHttpClient(object): destination, e ) - raise SynapseError(400, "Domain specified not found.") + raise logger.warn( "Sending request failed to %s: %s %s: %s - %s", -- cgit 1.5.1 From 5358966a878aa543659c10be09f4bf58b1568f2f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 16:51:33 +0000 Subject: Use git aware version string in User-Agent and Server headers --- synapse/http/agent_name.py | 18 ------------------ synapse/http/client.py | 10 +++++----- synapse/http/matrixfederationclient.py | 4 ++-- synapse/http/server.py | 27 ++++++++++++++++++--------- tests/utils.py | 10 ++++++++-- 5 files changed, 33 insertions(+), 36 deletions(-) delete mode 100644 synapse/http/agent_name.py (limited to 'synapse/http') diff --git a/synapse/http/agent_name.py b/synapse/http/agent_name.py deleted file mode 100644 index d761890863..0000000000 --- a/synapse/http/agent_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse import __version__ - -AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii") diff --git a/synapse/http/client.py b/synapse/http/client.py index e46e7db146..22b7145ac1 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -14,7 +14,6 @@ # limitations under the License. from synapse.api.errors import CodeMessageException -from synapse.http.agent_name import AGENT_NAME from syutil.jsonutil import encode_canonical_json from twisted.internet import defer, reactor @@ -44,6 +43,7 @@ class SimpleHttpClient(object): # BrowserLikePolicyForHTTPS which will do regular cert validation # 'like a browser' self.agent = Agent(reactor) + self.version_string = hs.version_string @defer.inlineCallbacks def post_urlencoded_get_json(self, uri, args={}): @@ -55,7 +55,7 @@ class SimpleHttpClient(object): uri.encode("ascii"), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -108,7 +108,7 @@ class SimpleHttpClient(object): "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }) ) @@ -149,7 +149,7 @@ class SimpleHttpClient(object): "PUT", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], "Content-Type": ["application/json"] }), bodyProducer=FileBodyProducer(StringIO(json_str)) @@ -182,7 +182,7 @@ class CaptchaServerHttpClient(SimpleHttpClient): bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }) ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 49a4c7d9d3..9d483032a0 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -20,7 +20,6 @@ from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone -from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext @@ -80,6 +79,7 @@ class MatrixFederationHttpClient(object): self.server_name = hs.hostname self.agent = MatrixFederationHttpAgent(reactor) self.clock = hs.get_clock() + self.version_string = hs.version_string @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, @@ -87,7 +87,7 @@ class MatrixFederationHttpClient(object): query_bytes=b"", retry_on_dns_fail=True): """ Creates and sends a request to the given url """ - headers_dict[b"User-Agent"] = [AGENT_NAME] + headers_dict[b"User-Agent"] = [self.version_string] headers_dict[b"Host"] = [destination] url_bytes = urlparse.urlunparse( diff --git a/synapse/http/server.py b/synapse/http/server.py index 589c30c199..74a101a5d7 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -14,7 +14,6 @@ # limitations under the License. -from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError ) @@ -74,6 +73,7 @@ class JsonResource(HttpServer, resource.Resource): self.clock = hs.get_clock() self.path_regexs = {} + self.version_string = hs.version_string def register_path(self, method, path_pattern, callback): self.path_regexs.setdefault(method, []).append( @@ -189,9 +189,13 @@ class JsonResource(HttpServer, resource.Resource): return # TODO: Only enable CORS for the requests that need it. - respond_with_json(request, code, response_json_object, send_cors=True, - response_code_message=response_code_message, - pretty_print=self._request_user_agent_is_curl) + respond_with_json( + request, code, response_json_object, + send_cors=True, + response_code_message=response_code_message, + pretty_print=self._request_user_agent_is_curl, + version_string=self.version_string, + ) @staticmethod def _request_user_agent_is_curl(request): @@ -221,18 +225,23 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, - response_code_message=None, pretty_print=False): + response_code_message=None, pretty_print=False, + version_string=""): if not pretty_print: json_bytes = encode_pretty_printed_json(json_object) else: json_bytes = encode_canonical_json(json_object) - return respond_with_json_bytes(request, code, json_bytes, send_cors, - response_code_message=response_code_message) + return respond_with_json_bytes( + request, code, json_bytes, + send_cors=send_cors, + response_code_message=response_code_message, + version_string=version_string + ) def respond_with_json_bytes(request, code, json_bytes, send_cors=False, - response_code_message=None): + version_string="", response_code_message=None): """Sends encoded JSON in response to the given request. Args: @@ -246,7 +255,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") - request.setHeader(b"Server", AGENT_NAME) + request.setHeader(b"Server", version_string) request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) if send_cors: diff --git a/tests/utils.py b/tests/utils.py index 39895c739f..110b9f86b8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -46,10 +46,16 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): if datastore is None: db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() - hs = HomeServer(name, db_pool=db_pool, config=config, **kargs) + hs = HomeServer( + name, db_pool=db_pool, config=config, + version_string="Synapse/tests", + **kargs + ) else: hs = HomeServer( - name, db_pool=None, datastore=datastore, config=config, **kargs + name, db_pool=None, datastore=datastore, config=config, + version_string="Synapse/tests", + **kargs ) defer.returnValue(hs) -- cgit 1.5.1 From 3f1871021e4efd1bbc513790e5c82f09b4c04503 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 17:32:12 +0000 Subject: Make /keys/ return correct Server version --- synapse/http/server_key_resource.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/server_key_resource.py b/synapse/http/server_key_resource.py index 4fc491dc82..71e9a51f5c 100644 --- a/synapse/http/server_key_resource.py +++ b/synapse/http/server_key_resource.py @@ -50,6 +50,7 @@ class LocalKey(Resource): def __init__(self, hs): self.hs = hs + self.version_string = hs.version_string self.response_body = encode_canonical_json( self.response_json_object(hs.config) ) @@ -82,7 +83,10 @@ class LocalKey(Resource): return json_object def render_GET(self, request): - return respond_with_json_bytes(request, 200, self.response_body) + return respond_with_json_bytes( + request, 200, self.response_body, + version_string=self.version_string + ) def getChild(self, name, request): if name == '': -- cgit 1.5.1 From 2de5b14fe00d2337a3179aa19185e66350ba1a3e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Mar 2015 15:36:37 +0000 Subject: Fix bug which prevented the HS pushing events to the AS due to FrozenEvents --- synapse/http/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/http') diff --git a/synapse/http/client.py b/synapse/http/client.py index 22b7145ac1..b53a07aa2d 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -143,7 +143,7 @@ class SimpleHttpClient(object): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) - json_str = json.dumps(json_body) + json_str = encode_canonical_json(json_body) response = yield self.agent.request( "PUT", -- cgit 1.5.1 From 9d9d39536b8be554eec2b16b2846ad31ac643721 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 5 Mar 2015 16:24:13 +0000 Subject: Slightly reduce the insane amounts of indentation in main http server response path, by 'continue'ing around a non-match or falling through --- synapse/http/server.py | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) (limited to 'synapse/http') diff --git a/synapse/http/server.py b/synapse/http/server.py index 74a101a5d7..767c3ef79b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -124,27 +124,29 @@ class JsonResource(HttpServer, resource.Resource): # and path regex match for path_entry in self.path_regexs.get(request.method, []): m = path_entry.pattern.match(request.path) - if m: - # We found a match! Trigger callback and then return the - # returned response. We pass both the request and any - # matched groups from the regex to the callback. - - args = [ - urllib.unquote(u).decode("UTF-8") for u in m.groups() - ] - - logger.info( - "Received request: %s %s", - request.method, request.path - ) - - code, response = yield path_entry.callback( - request, - *args - ) - - self._send_response(request, code, response) - return + if not m: + continue + + # We found a match! Trigger callback and then return the + # returned response. We pass both the request and any + # matched groups from the regex to the callback. + + args = [ + urllib.unquote(u).decode("UTF-8") for u in m.groups() + ] + + logger.info( + "Received request: %s %s", + request.method, request.path + ) + + code, response = yield path_entry.callback( + request, + *args + ) + + self._send_response(request, code, response) + return # Huh. No one wanted to handle that? Fiiiiiine. Send 400. raise UnrecognizedRequestError() -- cgit 1.5.1