diff --git a/synapse/http/agent_name.py b/synapse/http/agent_name.py
deleted file mode 100644
index d761890863..0000000000
--- a/synapse/http/agent_name.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse import __version__
-
-AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii")
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 7793bab106..b53a07aa2d 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -13,8 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from synapse.api.errors import CodeMessageException
+from syutil.jsonutil import encode_canonical_json
-from synapse.http.agent_name import AGENT_NAME
from twisted.internet import defer, reactor
from twisted.web.client import (
Agent, readBody, FileBodyProducer, PartialDownloadError
@@ -23,7 +24,7 @@ from twisted.web.http_headers import Headers
from StringIO import StringIO
-import json
+import simplejson as json
import logging
import urllib
@@ -42,6 +43,7 @@ class SimpleHttpClient(object):
# BrowserLikePolicyForHTTPS which will do regular cert validation
# 'like a browser'
self.agent = Agent(reactor)
+ self.version_string = hs.version_string
@defer.inlineCallbacks
def post_urlencoded_get_json(self, uri, args={}):
@@ -53,7 +55,7 @@ class SimpleHttpClient(object):
uri.encode("ascii"),
headers=Headers({
b"Content-Type": [b"application/x-www-form-urlencoded"],
- b"User-Agent": [AGENT_NAME],
+ b"User-Agent": [self.version_string],
}),
bodyProducer=FileBodyProducer(StringIO(query_bytes))
)
@@ -63,8 +65,27 @@ class SimpleHttpClient(object):
defer.returnValue(json.loads(body))
@defer.inlineCallbacks
+ def post_json_get_json(self, uri, post_json):
+ json_str = encode_canonical_json(post_json)
+
+ logger.info("HTTP POST %s -> %s", json_str, uri)
+
+ response = yield self.agent.request(
+ "POST",
+ uri.encode("ascii"),
+ headers=Headers({
+ "Content-Type": ["application/json"]
+ }),
+ bodyProducer=FileBodyProducer(StringIO(json_str))
+ )
+
+ body = yield readBody(response)
+
+ defer.returnValue(json.loads(body))
+
+ @defer.inlineCallbacks
def get_json(self, uri, args={}):
- """ Get's some json from the given host and path
+ """ Gets some json from the given URI.
Args:
uri (str): The URI to request, not including query parameters
@@ -72,15 +93,13 @@ class SimpleHttpClient(object):
None.
**Note**: The value of each key is assumed to be an iterable
and *not* a string.
-
Returns:
- Deferred: Succeeds when we get *any* HTTP response.
-
- The result of the deferred is a tuple of `(code, response)`,
- where `response` is a dict representing the decoded JSON body.
+ Deferred: Succeeds when we get *any* 2xx HTTP response, with the
+ HTTP body as JSON.
+ Raises:
+ On a non-2xx HTTP response. The response body will be used as the
+ error message.
"""
-
- yield
if len(args):
query_bytes = urllib.urlencode(args, True)
uri = "%s?%s" % (uri, query_bytes)
@@ -89,13 +108,62 @@ class SimpleHttpClient(object):
"GET",
uri.encode("ascii"),
headers=Headers({
- b"User-Agent": [AGENT_NAME],
+ b"User-Agent": [self.version_string],
})
)
body = yield readBody(response)
- defer.returnValue(json.loads(body))
+ if 200 <= response.code < 300:
+ defer.returnValue(json.loads(body))
+ else:
+ # NB: This is explicitly not json.loads(body)'d because the contract
+ # of CodeMessageException is a *string* message. Callers can always
+ # load it into JSON if they want.
+ raise CodeMessageException(response.code, body)
+
+ @defer.inlineCallbacks
+ def put_json(self, uri, json_body, args={}):
+ """ Puts some json to the given URI.
+
+ Args:
+ uri (str): The URI to request, not including query parameters
+ json_body (dict): The JSON to put in the HTTP body,
+ args (dict): A dictionary used to create query strings, defaults to
+ None.
+ **Note**: The value of each key is assumed to be an iterable
+ and *not* a string.
+ Returns:
+ Deferred: Succeeds when we get *any* 2xx HTTP response, with the
+ HTTP body as JSON.
+ Raises:
+ On a non-2xx HTTP response.
+ """
+ if len(args):
+ query_bytes = urllib.urlencode(args, True)
+ uri = "%s?%s" % (uri, query_bytes)
+
+ json_str = encode_canonical_json(json_body)
+
+ response = yield self.agent.request(
+ "PUT",
+ uri.encode("ascii"),
+ headers=Headers({
+ b"User-Agent": [self.version_string],
+ "Content-Type": ["application/json"]
+ }),
+ bodyProducer=FileBodyProducer(StringIO(json_str))
+ )
+
+ body = yield readBody(response)
+
+ if 200 <= response.code < 300:
+ defer.returnValue(json.loads(body))
+ else:
+ # NB: This is explicitly not json.loads(body)'d because the contract
+ # of CodeMessageException is a *string* message. Callers can always
+ # load it into JSON if they want.
+ raise CodeMessageException(response.code, body)
class CaptchaServerHttpClient(SimpleHttpClient):
@@ -114,7 +182,7 @@ class CaptchaServerHttpClient(SimpleHttpClient):
bodyProducer=FileBodyProducer(StringIO(query_bytes)),
headers=Headers({
b"Content-Type": [b"application/x-www-form-urlencoded"],
- b"User-Agent": [AGENT_NAME],
+ b"User-Agent": [self.version_string],
})
)
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 1dda3ba2c7..7db001cc63 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -20,18 +20,19 @@ from twisted.web.client import readBody, _AgentBase, _URI
from twisted.web.http_headers import Headers
from twisted.web._newclient import ResponseDone
-from synapse.http.agent_name import AGENT_NAME
from synapse.http.endpoint import matrix_federation_endpoint
from synapse.util.async import sleep
from synapse.util.logcontext import PreserveLoggingContext
from syutil.jsonutil import encode_canonical_json
-from synapse.api.errors import CodeMessageException, SynapseError, Codes
+from synapse.api.errors import (
+ SynapseError, Codes, HttpResponseException,
+)
from syutil.crypto.jsonsign import sign_json
-import json
+import simplejson as json
import logging
import urllib
import urlparse
@@ -77,6 +78,8 @@ class MatrixFederationHttpClient(object):
self.signing_key = hs.config.signing_key[0]
self.server_name = hs.hostname
self.agent = MatrixFederationHttpAgent(reactor)
+ self.clock = hs.get_clock()
+ self.version_string = hs.version_string
@defer.inlineCallbacks
def _create_request(self, destination, method, path_bytes,
@@ -84,7 +87,7 @@ class MatrixFederationHttpClient(object):
query_bytes=b"", retry_on_dns_fail=True):
""" Creates and sends a request to the given url
"""
- headers_dict[b"User-Agent"] = [AGENT_NAME]
+ headers_dict[b"User-Agent"] = [self.version_string]
headers_dict[b"Host"] = [destination]
url_bytes = urlparse.urlunparse(
@@ -116,7 +119,7 @@ class MatrixFederationHttpClient(object):
try:
with PreserveLoggingContext():
- response = yield self.agent.request(
+ request_deferred = self.agent.request(
destination,
endpoint,
method,
@@ -127,6 +130,11 @@ class MatrixFederationHttpClient(object):
producer
)
+ response = yield self.clock.time_bound_deferred(
+ request_deferred,
+ time_out=60,
+ )
+
logger.debug("Got response to %s", method)
break
except Exception as e:
@@ -136,16 +144,16 @@ class MatrixFederationHttpClient(object):
destination,
e
)
- raise SynapseError(400, "Domain specified not found.")
+ raise
logger.warn(
- "Sending request failed to %s: %s %s : %s",
+ "Sending request failed to %s: %s %s: %s - %s",
destination,
method,
url_bytes,
- e
+ type(e).__name__,
+ _flatten_response_never_received(e),
)
- _print_ex(e)
if retries_left:
yield sleep(2 ** (5 - retries_left))
@@ -163,13 +171,13 @@ class MatrixFederationHttpClient(object):
)
if 200 <= response.code < 300:
- # We need to update the transactions table to say it was sent?
pass
else:
# :'(
# Update transactions table?
- raise CodeMessageException(
- response.code, response.phrase
+ body = yield readBody(response)
+ raise HttpResponseException(
+ response.code, response.phrase, body
)
defer.returnValue(response)
@@ -238,11 +246,66 @@ class MatrixFederationHttpClient(object):
headers_dict={"Content-Type": ["application/json"]},
)
+ if 200 <= response.code < 300:
+ # We need to update the transactions table to say it was sent?
+ c_type = response.headers.getRawHeaders("Content-Type")
+
+ if "application/json" not in c_type:
+ raise RuntimeError(
+ "Content-Type not application/json"
+ )
+
logger.debug("Getting resp body")
body = yield readBody(response)
logger.debug("Got resp body")
- defer.returnValue((response.code, body))
+ defer.returnValue(json.loads(body))
+
+ @defer.inlineCallbacks
+ def post_json(self, destination, path, data={}):
+ """ Sends the specifed json data using POST
+
+ Args:
+ destination (str): The remote server to send the HTTP request
+ to.
+ path (str): The HTTP path.
+ data (dict): A dict containing the data that will be used as
+ the request body. This will be encoded as JSON.
+
+ Returns:
+ Deferred: Succeeds when we get a 2xx HTTP response. The result
+ will be the decoded JSON body. On a 4xx or 5xx error response a
+ CodeMessageException is raised.
+ """
+
+ def body_callback(method, url_bytes, headers_dict):
+ self.sign_request(
+ destination, method, url_bytes, headers_dict, data
+ )
+ return _JsonProducer(data)
+
+ response = yield self._create_request(
+ destination.encode("ascii"),
+ "POST",
+ path.encode("ascii"),
+ body_callback=body_callback,
+ headers_dict={"Content-Type": ["application/json"]},
+ )
+
+ if 200 <= response.code < 300:
+ # We need to update the transactions table to say it was sent?
+ c_type = response.headers.getRawHeaders("Content-Type")
+
+ if "application/json" not in c_type:
+ raise RuntimeError(
+ "Content-Type not application/json"
+ )
+
+ logger.debug("Getting resp body")
+ body = yield readBody(response)
+ logger.debug("Got resp body")
+
+ defer.returnValue(json.loads(body))
@defer.inlineCallbacks
def get_json(self, destination, path, args={}, retry_on_dns_fail=True):
@@ -284,7 +347,18 @@ class MatrixFederationHttpClient(object):
retry_on_dns_fail=retry_on_dns_fail
)
+ if 200 <= response.code < 300:
+ # We need to update the transactions table to say it was sent?
+ c_type = response.headers.getRawHeaders("Content-Type")
+
+ if "application/json" not in c_type:
+ raise RuntimeError(
+ "Content-Type not application/json"
+ )
+
+ logger.debug("Getting resp body")
body = yield readBody(response)
+ logger.debug("Got resp body")
defer.returnValue(json.loads(body))
@@ -373,14 +447,6 @@ def _readBodyToFile(response, stream, max_size):
return d
-def _print_ex(e):
- if hasattr(e, "reasons") and e.reasons:
- for ex in e.reasons:
- _print_ex(ex)
- else:
- logger.warn(e)
-
-
class _JsonProducer(object):
""" Used by the twisted http client to create the HTTP body from json
"""
@@ -400,3 +466,13 @@ class _JsonProducer(object):
def stopProducing(self):
pass
+
+
+def _flatten_response_never_received(e):
+ if hasattr(e, "reasons"):
+ return ", ".join(
+ _flatten_response_never_received(f.value)
+ for f in e.reasons
+ )
+ else:
+ return "%s: %s" % (type(e).__name__, e.message,)
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 8015a22edf..767c3ef79b 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -14,9 +14,8 @@
# limitations under the License.
-from synapse.http.agent_name import AGENT_NAME
from synapse.api.errors import (
- cs_exception, SynapseError, CodeMessageException
+ cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError
)
from synapse.util.logcontext import LoggingContext
@@ -69,10 +68,12 @@ class JsonResource(HttpServer, resource.Resource):
_PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"])
- def __init__(self):
+ def __init__(self, hs):
resource.Resource.__init__(self)
+ self.clock = hs.get_clock()
self.path_regexs = {}
+ self.version_string = hs.version_string
def register_path(self, method, path_pattern, callback):
self.path_regexs.setdefault(method, []).append(
@@ -111,6 +112,8 @@ class JsonResource(HttpServer, resource.Resource):
This checks if anyone has registered a callback for that method and
path.
"""
+ code = None
+ start = self.clock.time_msec()
try:
# Just say yes to OPTIONS.
if request.method == "OPTIONS":
@@ -121,37 +124,42 @@ class JsonResource(HttpServer, resource.Resource):
# and path regex match
for path_entry in self.path_regexs.get(request.method, []):
m = path_entry.pattern.match(request.path)
- if m:
- # We found a match! Trigger callback and then return the
- # returned response. We pass both the request and any
- # matched groups from the regex to the callback.
+ if not m:
+ continue
- args = [
- urllib.unquote(u).decode("UTF-8") for u in m.groups()
- ]
+ # We found a match! Trigger callback and then return the
+ # returned response. We pass both the request and any
+ # matched groups from the regex to the callback.
- code, response = yield path_entry.callback(
- request,
- *args
- )
+ args = [
+ urllib.unquote(u).decode("UTF-8") for u in m.groups()
+ ]
- self._send_response(request, code, response)
- return
+ logger.info(
+ "Received request: %s %s",
+ request.method, request.path
+ )
+
+ code, response = yield path_entry.callback(
+ request,
+ *args
+ )
+
+ self._send_response(request, code, response)
+ return
# Huh. No one wanted to handle that? Fiiiiiine. Send 400.
- self._send_response(
- request,
- 400,
- {"error": "Unrecognized request"}
- )
+ raise UnrecognizedRequestError()
except CodeMessageException as e:
if isinstance(e, SynapseError):
logger.info("%s SynapseError: %s - %s", request, e.code, e.msg)
else:
logger.exception(e)
+
+ code = e.code
self._send_response(
request,
- e.code,
+ code,
cs_exception(e),
response_code_message=e.response_code_message
)
@@ -162,6 +170,14 @@ class JsonResource(HttpServer, resource.Resource):
500,
{"error": "Internal server error"}
)
+ finally:
+ code = str(code) if code else "-"
+
+ end = self.clock.time_msec()
+ logger.info(
+ "Processed request: %dms %s %s %s",
+ end-start, code, request.method, request.path
+ )
def _send_response(self, request, code, response_json_object,
response_code_message=None):
@@ -175,9 +191,13 @@ class JsonResource(HttpServer, resource.Resource):
return
# TODO: Only enable CORS for the requests that need it.
- respond_with_json(request, code, response_json_object, send_cors=True,
- response_code_message=response_code_message,
- pretty_print=self._request_user_agent_is_curl)
+ respond_with_json(
+ request, code, response_json_object,
+ send_cors=True,
+ response_code_message=response_code_message,
+ pretty_print=self._request_user_agent_is_curl,
+ version_string=self.version_string,
+ )
@staticmethod
def _request_user_agent_is_curl(request):
@@ -207,18 +227,23 @@ class RootRedirect(resource.Resource):
def respond_with_json(request, code, json_object, send_cors=False,
- response_code_message=None, pretty_print=False):
+ response_code_message=None, pretty_print=False,
+ version_string=""):
if not pretty_print:
json_bytes = encode_pretty_printed_json(json_object)
else:
json_bytes = encode_canonical_json(json_object)
- return respond_with_json_bytes(request, code, json_bytes, send_cors,
- response_code_message=response_code_message)
+ return respond_with_json_bytes(
+ request, code, json_bytes,
+ send_cors=send_cors,
+ response_code_message=response_code_message,
+ version_string=version_string
+ )
def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
- response_code_message=None):
+ version_string="", response_code_message=None):
"""Sends encoded JSON in response to the given request.
Args:
@@ -232,7 +257,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
request.setResponseCode(code, message=response_code_message)
request.setHeader(b"Content-Type", b"application/json")
- request.setHeader(b"Server", AGENT_NAME)
+ request.setHeader(b"Server", version_string)
request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))
if send_cors:
diff --git a/synapse/http/server_key_resource.py b/synapse/http/server_key_resource.py
index 4fc491dc82..71e9a51f5c 100644
--- a/synapse/http/server_key_resource.py
+++ b/synapse/http/server_key_resource.py
@@ -50,6 +50,7 @@ class LocalKey(Resource):
def __init__(self, hs):
self.hs = hs
+ self.version_string = hs.version_string
self.response_body = encode_canonical_json(
self.response_json_object(hs.config)
)
@@ -82,7 +83,10 @@ class LocalKey(Resource):
return json_object
def render_GET(self, request):
- return respond_with_json_bytes(request, 200, self.response_body)
+ return respond_with_json_bytes(
+ request, 200, self.response_body,
+ version_string=self.version_string
+ )
def getChild(self, name, request):
if name == '':
diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py
new file mode 100644
index 0000000000..a4eb6c817c
--- /dev/null
+++ b/synapse/http/servlet.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014, 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" This module contains base REST classes for constructing REST servlets. """
+
+from synapse.api.errors import SynapseError
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class RestServlet(object):
+
+ """ A Synapse REST Servlet.
+
+ An implementing class can either provide its own custom 'register' method,
+ or use the automatic pattern handling provided by the base class.
+
+ To use this latter, the implementing class instead provides a `PATTERN`
+ class attribute containing a pre-compiled regular expression. The automatic
+ register method will then use this method to register any of the following
+ instance methods associated with the corresponding HTTP method:
+
+ on_GET
+ on_PUT
+ on_POST
+ on_DELETE
+ on_OPTIONS
+
+ Automatically handles turning CodeMessageExceptions thrown by these methods
+ into the appropriate HTTP response.
+ """
+
+ def register(self, http_server):
+ """ Register this servlet with the given HTTP server. """
+ if hasattr(self, "PATTERN"):
+ pattern = self.PATTERN
+
+ for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"):
+ if hasattr(self, "on_%s" % (method)):
+ method_handler = getattr(self, "on_%s" % (method))
+ http_server.register_path(method, pattern, method_handler)
+ else:
+ raise NotImplementedError("RestServlet must register something.")
+
+ @staticmethod
+ def parse_integer(request, name, default=None, required=False):
+ if name in request.args:
+ try:
+ return int(request.args[name][0])
+ except:
+ message = "Query parameter %r must be an integer" % (name,)
+ raise SynapseError(400, message)
+ else:
+ if required:
+ message = "Missing integer query parameter %r" % (name,)
+ raise SynapseError(400, message)
+ else:
+ return default
+
+ @staticmethod
+ def parse_boolean(request, name, default=None, required=False):
+ if name in request.args:
+ try:
+ return {
+ "true": True,
+ "false": False,
+ }[request.args[name][0]]
+ except:
+ message = (
+ "Boolean query parameter %r must be one of"
+ " ['true', 'false']"
+ ) % (name,)
+ raise SynapseError(400, message)
+ else:
+ if required:
+ message = "Missing boolean query parameter %r" % (name,)
+ raise SynapseError(400, message)
+ else:
+ return default
+
+ @staticmethod
+ def parse_string(request, name, default=None, required=False,
+ allowed_values=None, param_type="string"):
+ if name in request.args:
+ value = request.args[name][0]
+ if allowed_values is not None and value not in allowed_values:
+ message = "Query parameter %r must be one of [%s]" % (
+ name, ", ".join(repr(v) for v in allowed_values)
+ )
+ raise SynapseError(message)
+ else:
+ return value
+ else:
+ if required:
+ message = "Missing %s query parameter %r" % (param_type, name)
+ raise SynapseError(400, message)
+ else:
+ return default
|
