diff --git a/synapse/http/client.py b/synapse/http/client.py
index 49737d55da..27e5190224 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -12,16 +12,18 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from OpenSSL import SSL
+from OpenSSL.SSL import VERIFY_NONE
from synapse.api.errors import CodeMessageException
from synapse.util.logcontext import preserve_context_over_fn
-from syutil.jsonutil import encode_canonical_json
import synapse.metrics
-from twisted.internet import defer, reactor
+from canonicaljson import encode_canonical_json
+
+from twisted.internet import defer, reactor, ssl
from twisted.web.client import (
Agent, readBody, FileBodyProducer, PartialDownloadError,
- HTTPConnectionPool,
)
from twisted.web.http_headers import Headers
@@ -56,10 +58,14 @@ class SimpleHttpClient(object):
# The default context factory in Twisted 14.0.0 (which we require) is
# BrowserLikePolicyForHTTPS which will do regular cert validation
# 'like a browser'
- pool = HTTPConnectionPool(reactor)
- pool.maxPersistentPerHost = 10
- self.agent = Agent(reactor, pool=pool)
- self.version_string = hs.version_string
+ self.agent = Agent(
+ reactor,
+ connectTimeout=15,
+ contextFactory=hs.get_http_client_context_factory()
+ )
+ self.user_agent = hs.version_string
+ if hs.config.user_agent_suffix:
+ self.user_agent = "%s %s" % (self.user_agent, hs.config.user_agent_suffix,)
def request(self, method, uri, *args, **kwargs):
# A small wrapper around self.agent.request() so we can easily attach
@@ -104,7 +110,7 @@ class SimpleHttpClient(object):
uri.encode("ascii"),
headers=Headers({
b"Content-Type": [b"application/x-www-form-urlencoded"],
- b"User-Agent": [self.version_string],
+ b"User-Agent": [self.user_agent],
}),
bodyProducer=FileBodyProducer(StringIO(query_bytes))
)
@@ -123,7 +129,8 @@ class SimpleHttpClient(object):
"POST",
uri.encode("ascii"),
headers=Headers({
- "Content-Type": ["application/json"]
+ b"Content-Type": [b"application/json"],
+ b"User-Agent": [self.user_agent],
}),
bodyProducer=FileBodyProducer(StringIO(json_str))
)
@@ -149,16 +156,40 @@ class SimpleHttpClient(object):
On a non-2xx HTTP response. The response body will be used as the
error message.
"""
+ body = yield self.get_raw(uri, args)
+ defer.returnValue(json.loads(body))
+
+ @defer.inlineCallbacks
+ def put_json(self, uri, json_body, args={}):
+ """ Puts some json to the given URI.
+
+ Args:
+ uri (str): The URI to request, not including query parameters
+ json_body (dict): The JSON to put in the HTTP body,
+ args (dict): A dictionary used to create query strings, defaults to
+ None.
+ **Note**: The value of each key is assumed to be an iterable
+ and *not* a string.
+ Returns:
+ Deferred: Succeeds when we get *any* 2xx HTTP response, with the
+ HTTP body as JSON.
+ Raises:
+ On a non-2xx HTTP response.
+ """
if len(args):
query_bytes = urllib.urlencode(args, True)
uri = "%s?%s" % (uri, query_bytes)
+ json_str = encode_canonical_json(json_body)
+
response = yield self.request(
- "GET",
+ "PUT",
uri.encode("ascii"),
headers=Headers({
- b"User-Agent": [self.version_string],
- })
+ b"User-Agent": [self.user_agent],
+ "Content-Type": ["application/json"]
+ }),
+ bodyProducer=FileBodyProducer(StringIO(json_str))
)
body = yield preserve_context_over_fn(readBody, response)
@@ -172,46 +203,39 @@ class SimpleHttpClient(object):
raise CodeMessageException(response.code, body)
@defer.inlineCallbacks
- def put_json(self, uri, json_body, args={}):
- """ Puts some json to the given URI.
+ def get_raw(self, uri, args={}):
+ """ Gets raw text from the given URI.
Args:
uri (str): The URI to request, not including query parameters
- json_body (dict): The JSON to put in the HTTP body,
args (dict): A dictionary used to create query strings, defaults to
None.
**Note**: The value of each key is assumed to be an iterable
and *not* a string.
Returns:
Deferred: Succeeds when we get *any* 2xx HTTP response, with the
- HTTP body as JSON.
+ HTTP body at text.
Raises:
- On a non-2xx HTTP response.
+ On a non-2xx HTTP response. The response body will be used as the
+ error message.
"""
if len(args):
query_bytes = urllib.urlencode(args, True)
uri = "%s?%s" % (uri, query_bytes)
- json_str = encode_canonical_json(json_body)
-
response = yield self.request(
- "PUT",
+ "GET",
uri.encode("ascii"),
headers=Headers({
- b"User-Agent": [self.version_string],
- "Content-Type": ["application/json"]
- }),
- bodyProducer=FileBodyProducer(StringIO(json_str))
+ b"User-Agent": [self.user_agent],
+ })
)
body = yield preserve_context_over_fn(readBody, response)
if 200 <= response.code < 300:
- defer.returnValue(json.loads(body))
+ defer.returnValue(body)
else:
- # NB: This is explicitly not json.loads(body)'d because the contract
- # of CodeMessageException is a *string* message. Callers can always
- # load it into JSON if they want.
raise CodeMessageException(response.code, body)
@@ -233,7 +257,7 @@ class CaptchaServerHttpClient(SimpleHttpClient):
bodyProducer=FileBodyProducer(StringIO(query_bytes)),
headers=Headers({
b"Content-Type": [b"application/x-www-form-urlencoded"],
- b"User-Agent": [self.version_string],
+ b"User-Agent": [self.user_agent],
})
)
@@ -251,3 +275,18 @@ def _print_ex(e):
_print_ex(ex)
else:
logger.exception(e)
+
+
+class InsecureInterceptableContextFactory(ssl.ContextFactory):
+ """
+ Factory for PyOpenSSL SSL contexts which accepts any certificate for any domain.
+
+ Do not use this since it allows an attacker to intercept your communications.
+ """
+
+ def __init__(self):
+ self._context = SSL.Context(SSL.SSLv23_METHOD)
+ self._context.set_verify(VERIFY_NONE, lambda *_: None)
+
+ def getContext(self, hostname, port):
+ return self._context
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 854e17a473..ca9591556d 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -25,16 +25,17 @@ from synapse.util.async import sleep
from synapse.util.logcontext import preserve_context_over_fn
import synapse.metrics
-from syutil.jsonutil import encode_canonical_json
+from canonicaljson import encode_canonical_json
from synapse.api.errors import (
SynapseError, Codes, HttpResponseException,
)
-from syutil.crypto.jsonsign import sign_json
+from signedjson.sign import sign_json
import simplejson as json
import logging
+import random
import sys
import urllib
import urlparse
@@ -55,16 +56,19 @@ incoming_responses_counter = metrics.register_counter(
)
+MAX_RETRIES = 10
+
+
class MatrixFederationEndpointFactory(object):
def __init__(self, hs):
- self.tls_context_factory = hs.tls_context_factory
+ self.tls_server_context_factory = hs.tls_server_context_factory
def endpointForURI(self, uri):
destination = uri.netloc
return matrix_federation_endpoint(
reactor, destination, timeout=10,
- ssl_context_factory=self.tls_context_factory
+ ssl_context_factory=self.tls_server_context_factory
)
@@ -119,7 +123,7 @@ class MatrixFederationHttpClient(object):
# XXX: Would be much nicer to retry only at the transaction-layer
# (once we have reliable transactions in place)
- retries_left = 5
+ retries_left = MAX_RETRIES
http_url_bytes = urlparse.urlunparse(
("", "", path_bytes, param_bytes, query_bytes, "")
@@ -180,7 +184,10 @@ class MatrixFederationHttpClient(object):
)
if retries_left and not timeout:
- yield sleep(2 ** (5 - retries_left))
+ delay = 4 ** (MAX_RETRIES + 1 - retries_left)
+ delay = max(delay, 60)
+ delay *= random.uniform(0.8, 1.4)
+ yield sleep(delay)
retries_left -= 1
else:
raise
diff --git a/synapse/http/server.py b/synapse/http/server.py
index b60e905a62..50feea6f1c 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -21,8 +21,8 @@ from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
import synapse.metrics
import synapse.events
-from syutil.jsonutil import (
- encode_canonical_json, encode_pretty_printed_json, encode_json
+from canonicaljson import (
+ encode_canonical_json, encode_pretty_printed_json
)
from twisted.internet import defer
@@ -33,6 +33,7 @@ from twisted.web.util import redirectTo
import collections
import logging
import urllib
+import ujson
logger = logging.getLogger(__name__)
@@ -270,12 +271,11 @@ def respond_with_json(request, code, json_object, send_cors=False,
if pretty_print:
json_bytes = encode_pretty_printed_json(json_object) + "\n"
else:
- if canonical_json:
+ if canonical_json or synapse.events.USE_FROZEN_DICTS:
json_bytes = encode_canonical_json(json_object)
else:
- json_bytes = encode_json(
- json_object, using_frozen_dicts=synapse.events.USE_FROZEN_DICTS
- )
+ # ujson doesn't like frozen_dicts.
+ json_bytes = ujson.dumps(json_object, ensure_ascii=False)
return respond_with_json_bytes(
request, code, json_bytes,
|
