Merge branch 'release-v0.7.0' of github.com:matrix-org/synapse v0.7.0

4 files changed, 238 insertions, 17 deletions

diff --git a/synapse/http/client.py b/synapse/http/client.py
index 7793bab106..43b2329780 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -15,6 +15,8 @@
 
 
 from synapse.http.agent_name import AGENT_NAME
+from syutil.jsonutil import encode_canonical_json
+
 from twisted.internet import defer, reactor
 from twisted.web.client import (
     Agent, readBody, FileBodyProducer, PartialDownloadError
@@ -23,7 +25,7 @@ from twisted.web.http_headers import Headers
 
 from StringIO import StringIO
 
-import json
+import simplejson as json
 import logging
 import urllib
 
@@ -63,6 +65,25 @@ class SimpleHttpClient(object):
         defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
+    def post_json_get_json(self, uri, post_json):
+        json_str = encode_canonical_json(post_json)
+
+        logger.info("HTTP POST %s -> %s", json_str, uri)
+
+        response = yield self.agent.request(
+            "POST",
+            uri.encode("ascii"),
+            headers=Headers({
+                "Content-Type": ["application/json"]
+            }),
+            bodyProducer=FileBodyProducer(StringIO(json_str))
+        )
+
+        body = yield readBody(response)
+
+        defer.returnValue(json.loads(body))
+
+    @defer.inlineCallbacks
     def get_json(self, uri, args={}):
         """ Get's some json from the given host and path
 
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index aa14782b0f..1927948001 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -27,11 +27,13 @@ from synapse.util.logcontext import PreserveLoggingContext
 
 from syutil.jsonutil import encode_canonical_json
 
-from synapse.api.errors import CodeMessageException, SynapseError, Codes
+from synapse.api.errors import (
+    SynapseError, Codes, HttpResponseException,
+)
 
 from syutil.crypto.jsonsign import sign_json
 
-import json
+import simplejson as json
 import logging
 import urllib
 import urlparse
@@ -72,12 +74,12 @@ class MatrixFederationHttpClient(object):
             requests.
     """
 
-
     def __init__(self, hs):
         self.hs = hs
         self.signing_key = hs.config.signing_key[0]
         self.server_name = hs.hostname
         self.agent = MatrixFederationHttpAgent(reactor)
+        self.clock = hs.get_clock()
 
     @defer.inlineCallbacks
     def _create_request(self, destination, method, path_bytes,
@@ -117,7 +119,7 @@ class MatrixFederationHttpClient(object):
 
             try:
                 with PreserveLoggingContext():
-                    response = yield self.agent.request(
+                    request_deferred = self.agent.request(
                         destination,
                         endpoint,
                         method,
@@ -128,6 +130,11 @@ class MatrixFederationHttpClient(object):
                         producer
                     )
 
+                    response = yield self.clock.time_bound_deferred(
+                        request_deferred,
+                        time_out=60,
+                    )
+
                 logger.debug("Got response to %s", method)
                 break
             except Exception as e:
@@ -164,13 +171,13 @@ class MatrixFederationHttpClient(object):
         )
 
         if 200 <= response.code < 300:
-            # We need to update the transactions table to say it was sent?
             pass
         else:
             # :'(
             # Update transactions table?
-            raise CodeMessageException(
-                response.code, response.phrase
+            body = yield readBody(response)
+            raise HttpResponseException(
+                response.code, response.phrase, body
             )
 
         defer.returnValue(response)
@@ -239,11 +246,66 @@ class MatrixFederationHttpClient(object):
             headers_dict={"Content-Type": ["application/json"]},
         )
 
+        if 200 <= response.code < 300:
+            # We need to update the transactions table to say it was sent?
+            c_type = response.headers.getRawHeaders("Content-Type")
+
+            if "application/json" not in c_type:
+                raise RuntimeError(
+                    "Content-Type not application/json"
+                )
+
+        logger.debug("Getting resp body")
+        body = yield readBody(response)
+        logger.debug("Got resp body")
+
+        defer.returnValue(json.loads(body))
+
+    @defer.inlineCallbacks
+    def post_json(self, destination, path, data={}):
+        """ Sends the specifed json data using POST
+
+        Args:
+            destination (str): The remote server to send the HTTP request
+                to.
+            path (str): The HTTP path.
+            data (dict): A dict containing the data that will be used as
+                the request body. This will be encoded as JSON.
+
+        Returns:
+            Deferred: Succeeds when we get a 2xx HTTP response. The result
+            will be the decoded JSON body. On a 4xx or 5xx error response a
+            CodeMessageException is raised.
+        """
+
+        def body_callback(method, url_bytes, headers_dict):
+            self.sign_request(
+                destination, method, url_bytes, headers_dict, data
+            )
+            return _JsonProducer(data)
+
+        response = yield self._create_request(
+            destination.encode("ascii"),
+            "POST",
+            path.encode("ascii"),
+            body_callback=body_callback,
+            headers_dict={"Content-Type": ["application/json"]},
+        )
+
+        if 200 <= response.code < 300:
+            # We need to update the transactions table to say it was sent?
+            c_type = response.headers.getRawHeaders("Content-Type")
+
+            if "application/json" not in c_type:
+                raise RuntimeError(
+                    "Content-Type not application/json"
+                )
+
         logger.debug("Getting resp body")
         body = yield readBody(response)
         logger.debug("Got resp body")
 
-        defer.returnValue((response.code, body))
+        defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
     def get_json(self, destination, path, args={}, retry_on_dns_fail=True):
@@ -285,7 +347,18 @@ class MatrixFederationHttpClient(object):
             retry_on_dns_fail=retry_on_dns_fail
         )
 
+        if 200 <= response.code < 300:
+            # We need to update the transactions table to say it was sent?
+            c_type = response.headers.getRawHeaders("Content-Type")
+
+            if "application/json" not in c_type:
+                raise RuntimeError(
+                    "Content-Type not application/json"
+                )
+
+        logger.debug("Getting resp body")
         body = yield readBody(response)
+        logger.debug("Got resp body")
 
         defer.returnValue(json.loads(body))
 
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 8015a22edf..589c30c199 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -16,7 +16,7 @@
 
 from synapse.http.agent_name import AGENT_NAME
 from synapse.api.errors import (
-    cs_exception, SynapseError, CodeMessageException
+    cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError
 )
 from synapse.util.logcontext import LoggingContext
 
@@ -69,9 +69,10 @@ class JsonResource(HttpServer, resource.Resource):
 
     _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"])
 
-    def __init__(self):
+    def __init__(self, hs):
         resource.Resource.__init__(self)
 
+        self.clock = hs.get_clock()
         self.path_regexs = {}
 
     def register_path(self, method, path_pattern, callback):
@@ -111,6 +112,8 @@ class JsonResource(HttpServer, resource.Resource):
             This checks if anyone has registered a callback for that method and
             path.
         """
+        code = None
+        start = self.clock.time_msec()
         try:
             # Just say yes to OPTIONS.
             if request.method == "OPTIONS":
@@ -130,6 +133,11 @@ class JsonResource(HttpServer, resource.Resource):
                         urllib.unquote(u).decode("UTF-8") for u in m.groups()
                     ]
 
+                    logger.info(
+                        "Received request: %s %s",
+                        request.method, request.path
+                    )
+
                     code, response = yield path_entry.callback(
                         request,
                         *args
@@ -139,19 +147,17 @@ class JsonResource(HttpServer, resource.Resource):
                     return
 
             # Huh. No one wanted to handle that? Fiiiiiine. Send 400.
-            self._send_response(
-                request,
-                400,
-                {"error": "Unrecognized request"}
-            )
+            raise UnrecognizedRequestError()
         except CodeMessageException as e:
             if isinstance(e, SynapseError):
                 logger.info("%s SynapseError: %s - %s", request, e.code, e.msg)
             else:
                 logger.exception(e)
+
+            code = e.code
             self._send_response(
                 request,
-                e.code,
+                code,
                 cs_exception(e),
                 response_code_message=e.response_code_message
             )
@@ -162,6 +168,14 @@ class JsonResource(HttpServer, resource.Resource):
                 500,
                 {"error": "Internal server error"}
             )
+        finally:
+            code = str(code) if code else "-"
+
+            end = self.clock.time_msec()
+            logger.info(
+                "Processed request: %dms %s %s %s",
+                end-start, code, request.method, request.path
+            )
 
     def _send_response(self, request, code, response_json_object,
                        response_code_message=None):
diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py
new file mode 100644
index 0000000000..a4eb6c817c
--- /dev/null
+++ b/synapse/http/servlet.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014, 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" This module contains base REST classes for constructing REST servlets. """
+
+from synapse.api.errors import SynapseError
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class RestServlet(object):
+
+    """ A Synapse REST Servlet.
+
+    An implementing class can either provide its own custom 'register' method,
+    or use the automatic pattern handling provided by the base class.
+
+    To use this latter, the implementing class instead provides a `PATTERN`
+    class attribute containing a pre-compiled regular expression. The automatic
+    register method will then use this method to register any of the following
+    instance methods associated with the corresponding HTTP method:
+
+      on_GET
+      on_PUT
+      on_POST
+      on_DELETE
+      on_OPTIONS
+
+    Automatically handles turning CodeMessageExceptions thrown by these methods
+    into the appropriate HTTP response.
+    """
+
+    def register(self, http_server):
+        """ Register this servlet with the given HTTP server. """
+        if hasattr(self, "PATTERN"):
+            pattern = self.PATTERN
+
+            for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"):
+                if hasattr(self, "on_%s" % (method)):
+                    method_handler = getattr(self, "on_%s" % (method))
+                    http_server.register_path(method, pattern, method_handler)
+        else:
+            raise NotImplementedError("RestServlet must register something.")
+
+    @staticmethod
+    def parse_integer(request, name, default=None, required=False):
+        if name in request.args:
+            try:
+                return int(request.args[name][0])
+            except:
+                message = "Query parameter %r must be an integer" % (name,)
+                raise SynapseError(400, message)
+        else:
+            if required:
+                message = "Missing integer query parameter %r" % (name,)
+                raise SynapseError(400, message)
+            else:
+                return default
+
+    @staticmethod
+    def parse_boolean(request, name, default=None, required=False):
+        if name in request.args:
+            try:
+                return {
+                    "true": True,
+                    "false": False,
+                }[request.args[name][0]]
+            except:
+                message = (
+                    "Boolean query parameter %r must be one of"
+                    " ['true', 'false']"
+                ) % (name,)
+                raise SynapseError(400, message)
+        else:
+            if required:
+                message = "Missing boolean query parameter %r" % (name,)
+                raise SynapseError(400, message)
+            else:
+                return default
+
+    @staticmethod
+    def parse_string(request, name, default=None, required=False,
+                     allowed_values=None, param_type="string"):
+        if name in request.args:
+            value = request.args[name][0]
+            if allowed_values is not None and value not in allowed_values:
+                message = "Query parameter %r must be one of [%s]" % (
+                    name, ", ".join(repr(v) for v in allowed_values)
+                )
+                raise SynapseError(message)
+            else:
+                return value
+        else:
+            if required:
+                message = "Missing %s query parameter %r" % (param_type, name)
+                raise SynapseError(400, message)
+            else:
+                return default