From 96d4bf90120a07faa5163c2e5af542358554dd61 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:07:31 +0000 Subject: Modify API for SimpleHttpClient.get_json and update usages. Previously, this would only return the HTTP body as JSON, and discard other response information (e.g. the HTTP response code). This has now been changed to throw a CodeMessageException on a non-2xx response, with the response code and body, which can then be parsed as JSON. Affected modules include: - Registration/Login (when using an email for IS auth) --- synapse/http/client.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/http/client.py b/synapse/http/client.py index 198f575cfa..5f4558be47 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from synapse.api.errors import CodeMessageException from synapse.http.agent_name import AGENT_NAME from twisted.internet import defer, reactor from twisted.web.client import ( @@ -83,7 +83,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def get_json(self, uri, args={}): - """ Get's some json from the given host and path + """ Gets some json from the given host and path Args: uri (str): The URI to request, not including query parameters @@ -91,15 +91,11 @@ class SimpleHttpClient(object): None. **Note**: The value of each key is assumed to be an iterable and *not* a string. - Returns: - Deferred: Succeeds when we get *any* HTTP response. - - The result of the deferred is a tuple of `(code, response)`, - where `response` is a dict representing the decoded JSON body. + Deferred: Succeeds when we get *any* 2xx HTTP response. + Raises: + On a non-2xx HTTP response. """ - - yield if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) @@ -114,7 +110,10 @@ class SimpleHttpClient(object): body = yield readBody(response) - defer.returnValue(json.loads(body)) + if 200 <= response.code < 300: + defer.returnValue(json.loads(body)) + else: + raise CodeMessageException(response.code, body) class CaptchaServerHttpClient(SimpleHttpClient): -- cgit 1.4.1 From 6d3e4f4d0aad4ad9b44b28349838ff48aef39440 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:32:44 +0000 Subject: Update user/alias query APIs to use new format of SimpleHttpClient.get_json --- synapse/appservice/api.py | 15 +++++++-------- synapse/http/client.py | 3 +++ 2 files changed, 10 insertions(+), 8 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 799ada96df..74508ecddf 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from twisted.internet import defer -from twisted.web.client import PartialDownloadError +from synapse.api.errors import CodeMessageException from synapse.http.client import SimpleHttpClient import logging @@ -42,11 +42,11 @@ class ApplicationServiceApi(SimpleHttpClient): }) if response: # just an empty json object defer.returnValue(True) - except PartialDownloadError as e: - if e.status == 404: + except CodeMessageException as e: + if e.code == 404: defer.returnValue(False) return - logger.warning("query_user to %s received %s", (uri, e.status)) + logger.warning("query_user to %s received %s", uri, e.code) @defer.inlineCallbacks def query_alias(self, service, alias): @@ -56,14 +56,13 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": self.hs_token }) - logger.info("%s", response[0]) if response: # just an empty json object defer.returnValue(True) - except PartialDownloadError as e: - if e.status == 404: + except CodeMessageException as e: + if e.code == 404: defer.returnValue(False) return - logger.warning("query_alias to %s received %s", (uri, e.status)) + logger.warning("query_alias to %s received %s", uri, e.code) def push_bulk(self, service, events): pass diff --git a/synapse/http/client.py b/synapse/http/client.py index 5f4558be47..fee8c901a2 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -113,6 +113,9 @@ class SimpleHttpClient(object): if 200 <= response.code < 300: defer.returnValue(json.loads(body)) else: + # NB: This is explicitly not json.loads(body)'d because the contract + # of CodeMessageException is a *string* message. Callers can always + # load it into JSON if they want. raise CodeMessageException(response.code, body) -- cgit 1.4.1 From 543e84fe70bdc0af6be5feb237de2d18adf352ab Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:39:51 +0000 Subject: Add SimpleHttpClient.put_json with the same semantics as get_json. --- synapse/http/client.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/http/client.py b/synapse/http/client.py index fee8c901a2..575510637e 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -83,7 +83,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def get_json(self, uri, args={}): - """ Gets some json from the given host and path + """ Gets some json from the given URI. Args: uri (str): The URI to request, not including query parameters @@ -92,7 +92,8 @@ class SimpleHttpClient(object): **Note**: The value of each key is assumed to be an iterable and *not* a string. Returns: - Deferred: Succeeds when we get *any* 2xx HTTP response. + Deferred: Succeeds when we get *any* 2xx HTTP response, with the + HTTP body as JSON. Raises: On a non-2xx HTTP response. """ @@ -118,6 +119,49 @@ class SimpleHttpClient(object): # load it into JSON if they want. raise CodeMessageException(response.code, body) + @defer.inlineCallbacks + def put_json(self, uri, json_body, args={}): + """ Puts some json to the given URI. + + Args: + uri (str): The URI to request, not including query parameters + json_body (dict): The JSON to put in the HTTP body, + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + Returns: + Deferred: Succeeds when we get *any* 2xx HTTP response, with the + HTTP body as JSON. + Raises: + On a non-2xx HTTP response. + """ + if len(args): + query_bytes = urllib.urlencode(args, True) + uri = "%s?%s" % (uri, query_bytes) + + json_str = json.dumps(json_body) + + response = yield self.agent.request( + "PUT", + uri.encode("ascii"), + headers=Headers({ + b"User-Agent": [AGENT_NAME], + "Content-Type": ["application/json"] + }), + bodyProducer=FileBodyProducer(StringIO(json_str)) + ) + + body = yield readBody(response) + + if 200 <= response.code < 300: + defer.returnValue(json.loads(body)) + else: + # NB: This is explicitly not json.loads(body)'d because the contract + # of CodeMessageException is a *string* message. Callers can always + # load it into JSON if they want. + raise CodeMessageException(response.code, body) + class CaptchaServerHttpClient(SimpleHttpClient): """ -- cgit 1.4.1 From f51832442674a1f72c41ffce2279880109fc7ff0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 16:41:16 +0000 Subject: Minor tweaks based on PR feedback. --- synapse/appservice/api.py | 6 +++--- synapse/http/client.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'synapse/http/client.py') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 6192813c03..c2179f8d55 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -80,11 +80,11 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.put_json( - uri, - { + uri=uri, + json_body={ "events": events }, - { + args={ "access_token": service.hs_token }) if response: # just an empty json object diff --git a/synapse/http/client.py b/synapse/http/client.py index 575510637e..7b23116556 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -95,7 +95,8 @@ class SimpleHttpClient(object): Deferred: Succeeds when we get *any* 2xx HTTP response, with the HTTP body as JSON. Raises: - On a non-2xx HTTP response. + On a non-2xx HTTP response. The response body will be used as the + error message. """ if len(args): query_bytes = urllib.urlencode(args, True) -- cgit 1.4.1 From 5358966a878aa543659c10be09f4bf58b1568f2f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 16:51:33 +0000 Subject: Use git aware version string in User-Agent and Server headers --- synapse/http/agent_name.py | 18 ------------------ synapse/http/client.py | 10 +++++----- synapse/http/matrixfederationclient.py | 4 ++-- synapse/http/server.py | 27 ++++++++++++++++++--------- tests/utils.py | 10 ++++++++-- 5 files changed, 33 insertions(+), 36 deletions(-) delete mode 100644 synapse/http/agent_name.py (limited to 'synapse/http/client.py') diff --git a/synapse/http/agent_name.py b/synapse/http/agent_name.py deleted file mode 100644 index d761890863..0000000000 --- a/synapse/http/agent_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse import __version__ - -AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii") diff --git a/synapse/http/client.py b/synapse/http/client.py index e46e7db146..22b7145ac1 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -14,7 +14,6 @@ # limitations under the License. from synapse.api.errors import CodeMessageException -from synapse.http.agent_name import AGENT_NAME from syutil.jsonutil import encode_canonical_json from twisted.internet import defer, reactor @@ -44,6 +43,7 @@ class SimpleHttpClient(object): # BrowserLikePolicyForHTTPS which will do regular cert validation # 'like a browser' self.agent = Agent(reactor) + self.version_string = hs.version_string @defer.inlineCallbacks def post_urlencoded_get_json(self, uri, args={}): @@ -55,7 +55,7 @@ class SimpleHttpClient(object): uri.encode("ascii"), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -108,7 +108,7 @@ class SimpleHttpClient(object): "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }) ) @@ -149,7 +149,7 @@ class SimpleHttpClient(object): "PUT", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], "Content-Type": ["application/json"] }), bodyProducer=FileBodyProducer(StringIO(json_str)) @@ -182,7 +182,7 @@ class CaptchaServerHttpClient(SimpleHttpClient): bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }) ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 49a4c7d9d3..9d483032a0 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -20,7 +20,6 @@ from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone -from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext @@ -80,6 +79,7 @@ class MatrixFederationHttpClient(object): self.server_name = hs.hostname self.agent = MatrixFederationHttpAgent(reactor) self.clock = hs.get_clock() + self.version_string = hs.version_string @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, @@ -87,7 +87,7 @@ class MatrixFederationHttpClient(object): query_bytes=b"", retry_on_dns_fail=True): """ Creates and sends a request to the given url """ - headers_dict[b"User-Agent"] = [AGENT_NAME] + headers_dict[b"User-Agent"] = [self.version_string] headers_dict[b"Host"] = [destination] url_bytes = urlparse.urlunparse( diff --git a/synapse/http/server.py b/synapse/http/server.py index 589c30c199..74a101a5d7 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -14,7 +14,6 @@ # limitations under the License. -from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError ) @@ -74,6 +73,7 @@ class JsonResource(HttpServer, resource.Resource): self.clock = hs.get_clock() self.path_regexs = {} + self.version_string = hs.version_string def register_path(self, method, path_pattern, callback): self.path_regexs.setdefault(method, []).append( @@ -189,9 +189,13 @@ class JsonResource(HttpServer, resource.Resource): return # TODO: Only enable CORS for the requests that need it. - respond_with_json(request, code, response_json_object, send_cors=True, - response_code_message=response_code_message, - pretty_print=self._request_user_agent_is_curl) + respond_with_json( + request, code, response_json_object, + send_cors=True, + response_code_message=response_code_message, + pretty_print=self._request_user_agent_is_curl, + version_string=self.version_string, + ) @staticmethod def _request_user_agent_is_curl(request): @@ -221,18 +225,23 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, - response_code_message=None, pretty_print=False): + response_code_message=None, pretty_print=False, + version_string=""): if not pretty_print: json_bytes = encode_pretty_printed_json(json_object) else: json_bytes = encode_canonical_json(json_object) - return respond_with_json_bytes(request, code, json_bytes, send_cors, - response_code_message=response_code_message) + return respond_with_json_bytes( + request, code, json_bytes, + send_cors=send_cors, + response_code_message=response_code_message, + version_string=version_string + ) def respond_with_json_bytes(request, code, json_bytes, send_cors=False, - response_code_message=None): + version_string="", response_code_message=None): """Sends encoded JSON in response to the given request. Args: @@ -246,7 +255,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") - request.setHeader(b"Server", AGENT_NAME) + request.setHeader(b"Server", version_string) request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) if send_cors: diff --git a/tests/utils.py b/tests/utils.py index 39895c739f..110b9f86b8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -46,10 +46,16 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): if datastore is None: db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() - hs = HomeServer(name, db_pool=db_pool, config=config, **kargs) + hs = HomeServer( + name, db_pool=db_pool, config=config, + version_string="Synapse/tests", + **kargs + ) else: hs = HomeServer( - name, db_pool=None, datastore=datastore, config=config, **kargs + name, db_pool=None, datastore=datastore, config=config, + version_string="Synapse/tests", + **kargs ) defer.returnValue(hs) -- cgit 1.4.1