Split PlainHttpClient into separate clients for talking to Identity servers and talking to Capatcha servers

5 files changed, 161 insertions, 154 deletions

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 2f1b954902..61d574a00f 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -25,7 +25,7 @@ from twisted.web.static import File
 from twisted.web.server import Site
 from synapse.http.server import JsonResource, RootRedirect
 from synapse.http.content_repository import ContentRepoResource
-from synapse.http.client import TwistedHttpClient
+from synapse.http.client import MatrixHttpClient
 from synapse.api.urls import (
     CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX
 )
@@ -47,7 +47,7 @@ logger = logging.getLogger(__name__)
 class SynapseHomeServer(HomeServer):
 
     def build_http_client(self):
-        return TwistedHttpClient(self)
+        return MatrixHttpClient(self)
 
     def build_resource_for_client(self):
         return JsonResource()
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index 84c3a1d56f..cec7737e09 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
 from ._base import BaseHandler
 
 from synapse.api.errors import SynapseError
-from synapse.http.client import HttpClient
+from synapse.http.client import MatrixHttpClient
 from synapse.api.events.room import RoomAliasesEvent
 
 import logging
@@ -98,7 +98,7 @@ class DirectoryHandler(BaseHandler):
                 query_type="directory",
                 args={
                     "room_alias": room_alias.to_string(),
-                    HttpClient.RETRY_DNS_LOOKUP_FAILURES: False
+                    MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False
                 }
             )
 
diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 80ffdd2726..3f152e18f0 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -17,7 +17,7 @@ from twisted.internet import defer
 
 from ._base import BaseHandler
 from synapse.api.errors import LoginError, Codes
-from synapse.http.client import PlainHttpClient
+from synapse.http.client import IdentityServerHttpClient
 from synapse.util.emailutils import EmailException
 import synapse.util.emailutils as emailutils
 
@@ -97,10 +97,10 @@ class LoginHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _query_email(self, email):
-        httpCli = PlainHttpClient(self.hs)
+        httpCli = IdentityServerHttpClient(self.hs)
         data = yield httpCli.get_json(
             'matrix.org:8090',  # TODO FIXME This should be configurable.
             "/_matrix/identity/api/v1/lookup?medium=email&address=" +
             "%s" % urllib.quote(email)
         )
-        defer.returnValue(data)
\ No newline at end of file
+        defer.returnValue(data)
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index a019d770d4..266495056e 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -22,7 +22,8 @@ from synapse.api.errors import (
 )
 from ._base import BaseHandler
 import synapse.util.stringutils as stringutils
-from synapse.http.client import PlainHttpClient
+from synapse.http.client import IdentityServerHttpClient
+from synapse.http.client import CaptchaServerHttpClient
 
 import base64
 import bcrypt
@@ -154,7 +155,9 @@ class RegistrationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _threepid_from_creds(self, creds):
-        httpCli = PlainHttpClient(self.hs)
+        # TODO: get this from the homeserver rather than creating a new one for
+        # each request
+        httpCli = IdentityServerHttpClient(self.hs)
         # XXX: make this configurable!
         trustedIdServers = ['matrix.org:8090']
         if not creds['idServer'] in trustedIdServers:
@@ -203,7 +206,9 @@ class RegistrationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _submit_captcha(self, ip_addr, private_key, challenge, response):
-        client = PlainHttpClient(self.hs)
+        # TODO: get this from the homeserver rather than creating a new one for
+        # each request
+        client = CaptchaServerHttpClient(self.hs)
         data = yield client.post_urlencoded_get_raw(
             "www.google.com:80",
             "/recaptcha/api/verify",
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 822afeec1d..e02cce5642 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -36,49 +36,6 @@ import urllib
 logger = logging.getLogger(__name__)
 
 
-class HttpClient(object):
-    """ Interface for talking json over http
-    """
-    RETRY_DNS_LOOKUP_FAILURES = "__retry_dns"
-
-    def put_json(self, destination, path, data):
-        """ Sends the specifed json data using PUT
-
-        Args:
-            destination (str): The remote server to send the HTTP request
-                to.
-            path (str): The HTTP path.
-            data (dict): A dict containing the data that will be used as
-                the request body. This will be encoded as JSON.
-
-        Returns:
-            Deferred: Succeeds when we get a 2xx HTTP response. The result
-            will be the decoded JSON body. On a 4xx or 5xx error response a
-            CodeMessageException is raised.
-        """
-        pass
-
-    def get_json(self, destination, path, args=None):
-        """ Get's some json from the given host homeserver and path
-
-        Args:
-            destination (str): The remote server to send the HTTP request
-                to.
-            path (str): The HTTP path.
-            args (dict): A dictionary used to create query strings, defaults to
-                None.
-                **Note**: The value of each key is assumed to be an iterable
-                and *not* a string.
-
-        Returns:
-            Deferred: Succeeds when we get *any* HTTP response.
-
-            The result of the deferred is a tuple of `(code, response)`,
-            where `response` is a dict representing the decoded JSON body.
-        """
-        pass
-
-
 class MatrixHttpAgent(_AgentBase):
 
     def __init__(self, reactor, pool=None):
@@ -102,23 +59,114 @@ class MatrixHttpAgent(_AgentBase):
                                          parsed_URI.originForm)
 
 
-class TwistedHttpClient(HttpClient):
-    """ Wrapper around the twisted HTTP client api.
+class BaseHttpClient(object):
+    """Base class for HTTP clients using twisted.
+    """
+
+    def __init__(self, hs):
+        self.agent = MatrixHttpAgent(reactor)
+        self.hs = hs
+
+    @defer.inlineCallbacks
+    def _create_request(self, destination, method, path_bytes, param_bytes=b"",
+                        query_bytes=b"", producer=None, headers_dict={},
+                        retry_on_dns_fail=True, on_send_callback=None):
+        """ Creates and sends a request to the given url
+        """
+        headers_dict[b"User-Agent"] = [b"Synapse"]
+        headers_dict[b"Host"] = [destination]
+
+        logger.debug("Sending request to %s: %s %s;%s?%s",
+                     destination, method, path_bytes, param_bytes, query_bytes)
+
+        logger.debug(
+            "Types: %s",
+            [
+                type(destination), type(method), type(path_bytes),
+                type(param_bytes),
+                type(query_bytes)
+            ]
+        )
+
+        retries_left = 5
+
+        endpoint = self._getEndpoint(reactor, destination);
+
+        while True:
+            if on_send_callback:
+                on_send_callback(destination, method, path_bytes, producer)
+
+            try:
+                response = yield self.agent.request(
+                    destination,
+                    endpoint,
+                    method,
+                    path_bytes,
+                    param_bytes,
+                    query_bytes,
+                    Headers(headers_dict),
+                    producer
+                )
+
+                logger.debug("Got response to %s", method)
+                break
+            except Exception as e:
+                if not retry_on_dns_fail and isinstance(e, DNSLookupError):
+                    logger.warn("DNS Lookup failed to %s with %s", destination,
+                                e)
+                    raise SynapseError(400, "Domain specified not found.")
+
+                logger.exception("Got error in _create_request")
+                _print_ex(e)
+
+                if retries_left:
+                    yield sleep(2 ** (5 - retries_left))
+                    retries_left -= 1
+                else:
+                    raise
+
+        if 200 <= response.code < 300:
+            # We need to update the transactions table to say it was sent?
+            pass
+        else:
+            # :'(
+            # Update transactions table?
+            logger.error(
+                "Got response %d %s", response.code, response.phrase
+            )
+            raise CodeMessageException(
+                response.code, response.phrase
+            )
+
+        defer.returnValue(response)
+
+
+class MatrixHttpClient(BaseHttpClient):
+    """ Wrapper around the twisted HTTP client api. Implements 
 
     Attributes:
         agent (twisted.web.client.Agent): The twisted Agent used to send the
             requests.
     """
 
-    def __init__(self, hs):
-        self.agent = MatrixHttpAgent(reactor)
-        self.hs = hs
+    RETRY_DNS_LOOKUP_FAILURES = "__retry_dns"
 
     @defer.inlineCallbacks
     def put_json(self, destination, path, data, on_send_callback=None):
-        if destination in _destination_mappings:
-            destination = _destination_mappings[destination]
+        """ Sends the specifed json data using PUT
 
+        Args:
+            destination (str): The remote server to send the HTTP request
+                to.
+            path (str): The HTTP path.
+            data (dict): A dict containing the data that will be used as
+                the request body. This will be encoded as JSON.
+
+        Returns:
+            Deferred: Succeeds when we get a 2xx HTTP response. The result
+            will be the decoded JSON body. On a 4xx or 5xx error response a
+            CodeMessageException is raised.
+        """
         response = yield self._create_request(
             destination.encode("ascii"),
             "PUT",
@@ -136,9 +184,23 @@ class TwistedHttpClient(HttpClient):
 
     @defer.inlineCallbacks
     def get_json(self, destination, path, args={}):
-        if destination in _destination_mappings:
-            destination = _destination_mappings[destination]
+        """ Get's some json from the given host homeserver and path
+
+        Args:
+            destination (str): The remote server to send the HTTP request
+                to.
+            path (str): The HTTP path.
+            args (dict): A dictionary used to create query strings, defaults to
+                None.
+                **Note**: The value of each key is assumed to be an iterable
+                and *not* a string.
+
+        Returns:
+            Deferred: Succeeds when we get *any* HTTP response.
 
+            The result of the deferred is a tuple of `(code, response)`,
+            where `response` is a dict representing the decoded JSON body.
+        """
         logger.debug("get_json args: %s", args)
 
         retry_on_dns_fail = True
@@ -163,6 +225,22 @@ class TwistedHttpClient(HttpClient):
 
         defer.returnValue(json.loads(body))
 
+
+    def _getEndpoint(self, reactor, destination):
+        return matrix_endpoint(
+            reactor, destination, timeout=10,
+            ssl_context_factory=self.hs.tls_context_factory
+        )
+
+
+class IdentityServerHttpClient(BaseHttpClient):
+    """Separate HTTP client for talking to the Identity servers since they
+    don't use SRV records and talk x-www-form-urlencoded rather than JSON.
+    """
+    def _getEndpoint(self, reactor, destination):
+        #TODO: This should be talking TLS
+        return matrix_endpoint(reactor, destination, timeout=10)
+
     @defer.inlineCallbacks
     def post_urlencoded_get_json(self, destination, path, args={}):
         if destination in _destination_mappings:
@@ -176,16 +254,25 @@ class TwistedHttpClient(HttpClient):
             "POST",
             path.encode("ascii"),
             producer=FileBodyProducer(StringIO(urllib.urlencode(args))),
-            headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]}
+            headers_dict={
+                "Content-Type": ["application/x-www-form-urlencoded"]
+            }
         )
 
         body = yield readBody(response)
 
         defer.returnValue(json.loads(body))
-        
-    # XXX FIXME : I'm so sorry.
+
+
+class CaptchaServerHttpClient(MatrixHttpClient):
+    """Separate HTTP client for talking to google's captcha servers"""
+
+    def _getEndpoint(self, reactor, destination):
+        return matrix_endpoint(reactor, destination, timeout=10)
+
     @defer.inlineCallbacks
-    def post_urlencoded_get_raw(self, destination, path, accept_partial=False, args={}):
+    def post_urlencoded_get_raw(self, destination, path, accept_partial=False,
+                                args={}):
         if destination in _destination_mappings:
             destination = _destination_mappings[destination]
 
@@ -196,7 +283,9 @@ class TwistedHttpClient(HttpClient):
             "POST",
             path.encode("ascii"),
             producer=FileBodyProducer(StringIO(urllib.urlencode(args))),
-            headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]}
+            headers_dict={
+                "Content-Type": ["application/x-www-form-urlencoded"]
+            }
         )
 
         try:
@@ -207,93 +296,6 @@ class TwistedHttpClient(HttpClient):
                 defer.returnValue(e.response)
             else:
                 raise e
-        
-
-    @defer.inlineCallbacks
-    def _create_request(self, destination, method, path_bytes, param_bytes=b"",
-                        query_bytes=b"", producer=None, headers_dict={},
-                        retry_on_dns_fail=True, on_send_callback=None):
-        """ Creates and sends a request to the given url
-        """
-        headers_dict[b"User-Agent"] = [b"Synapse"]
-        headers_dict[b"Host"] = [destination]
-
-        logger.debug("Sending request to %s: %s %s;%s?%s",
-                     destination, method, path_bytes, param_bytes, query_bytes)
-
-        logger.debug(
-            "Types: %s",
-            [
-                type(destination), type(method), type(path_bytes),
-                type(param_bytes),
-                type(query_bytes)
-            ]
-        )
-
-        retries_left = 5
-
-        # TODO: setup and pass in an ssl_context to enable TLS
-        endpoint = self._getEndpoint(reactor, destination);
-
-        while True:
-            if on_send_callback:
-                on_send_callback(destination, method, path_bytes, producer)
-
-            try:
-                response = yield self.agent.request(
-                    destination,
-                    endpoint,
-                    method,
-                    path_bytes,
-                    param_bytes,
-                    query_bytes,
-                    Headers(headers_dict),
-                    producer
-                )
-
-                logger.debug("Got response to %s", method)
-                break
-            except Exception as e:
-                if not retry_on_dns_fail and isinstance(e, DNSLookupError):
-                    logger.warn("DNS Lookup failed to %s with %s", destination,
-                                e)
-                    raise SynapseError(400, "Domain specified not found.")
-
-                logger.exception("Got error in _create_request")
-                _print_ex(e)
-
-                if retries_left:
-                    yield sleep(2 ** (5 - retries_left))
-                    retries_left -= 1
-                else:
-                    raise
-
-        if 200 <= response.code < 300:
-            # We need to update the transactions table to say it was sent?
-            pass
-        else:
-            # :'(
-            # Update transactions table?
-            logger.error(
-                "Got response %d %s", response.code, response.phrase
-            )
-            raise CodeMessageException(
-                response.code, response.phrase
-            )
-
-        defer.returnValue(response)
-
-    def _getEndpoint(self, reactor, destination):
-        return matrix_endpoint(
-            reactor, destination, timeout=10,
-            ssl_context_factory=self.hs.tls_context_factory
-        )
-
-
-class PlainHttpClient(TwistedHttpClient):
-    def _getEndpoint(self, reactor, destination):
-        return matrix_endpoint(reactor, destination, timeout=10)
-    
 
 def _print_ex(e):
     if hasattr(e, "reasons") and e.reasons: