diff --git a/README.rst b/README.rst
index 9a7c04b55e..8e22109973 100644
--- a/README.rst
+++ b/README.rst
@@ -199,6 +199,8 @@ by installing the ``libjemalloc1`` package and adding this line to
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
+This can make a significant difference on Python 2.7 - it's unclear how
+much of an improvement it provides on Python 3.x.
Upgrading an existing Synapse
=============================
diff --git a/changelog.d/4669.misc b/changelog.d/4669.misc
index 00a1a940ae..d5d0e27731 100644
--- a/changelog.d/4669.misc
+++ b/changelog.d/4669.misc
@@ -1 +1 @@
-Cleanup request exception logging
+Cleanup request exception logging.
diff --git a/changelog.d/4717.bugfix b/changelog.d/4717.bugfix
new file mode 100644
index 0000000000..79ab231477
--- /dev/null
+++ b/changelog.d/4717.bugfix
@@ -0,0 +1 @@
+Fix ACME config for python 2.
diff --git a/changelog.d/4718.bugfix b/changelog.d/4718.bugfix
new file mode 100644
index 0000000000..a7d1963ee1
--- /dev/null
+++ b/changelog.d/4718.bugfix
@@ -0,0 +1 @@
+Fix paginating over federation persisting incorrect state.
diff --git a/changelog.d/4721.feature b/changelog.d/4721.feature
new file mode 100644
index 0000000000..f932843ce7
--- /dev/null
+++ b/changelog.d/4721.feature
@@ -0,0 +1 @@
+Return correct error code when inviting a remote user to a room whose homeserver does not support the room version.
diff --git a/changelog.d/4722.misc b/changelog.d/4722.misc
new file mode 100644
index 0000000000..e9158c4dc2
--- /dev/null
+++ b/changelog.d/4722.misc
@@ -0,0 +1 @@
+Don't log exceptions when failing to fetch remote server keys
diff --git a/changelog.d/4737.misc b/changelog.d/4737.misc
new file mode 100644
index 0000000000..d5d0e27731
--- /dev/null
+++ b/changelog.d/4737.misc
@@ -0,0 +1 @@
+Cleanup request exception logging.
diff --git a/changelog.d/4738.misc b/changelog.d/4738.misc
new file mode 100644
index 0000000000..d5d0e27731
--- /dev/null
+++ b/changelog.d/4738.misc
@@ -0,0 +1 @@
+Cleanup request exception logging.
diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py
index 4064891ffb..d25196be08 100644
--- a/synapse/config/captcha.py
+++ b/synapse/config/captcha.py
@@ -47,5 +47,5 @@ class CaptchaConfig(Config):
#captcha_bypass_secret: "YOUR_SECRET_HERE"
# The API endpoint to use for verifying m.login.recaptcha responses.
- recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify"
+ recaptcha_siteverify_api: "https://www.recaptcha.net/recaptcha/api/siteverify"
"""
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 8d5d287357..40045de7ac 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -19,6 +19,8 @@ import warnings
from datetime import datetime
from hashlib import sha256
+import six
+
from unpaddedbase64 import encode_base64
from OpenSSL import crypto
@@ -36,9 +38,11 @@ class TlsConfig(Config):
acme_config = {}
self.acme_enabled = acme_config.get("enabled", False)
- self.acme_url = acme_config.get(
+
+ # hyperlink complains on py2 if this is not a Unicode
+ self.acme_url = six.text_type(acme_config.get(
"url", u"https://acme-v01.api.letsencrypt.org/directory"
- )
+ ))
self.acme_port = acme_config.get("port", 80)
self.acme_bind_addresses = acme_config.get("bind_addresses", ['::', '0.0.0.0'])
self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30)
@@ -55,7 +59,7 @@ class TlsConfig(Config):
)
if not self.tls_private_key_file:
raise ConfigError(
- "tls_certificate_path must be specified if TLS-enabled listeners are "
+ "tls_private_key_path must be specified if TLS-enabled listeners are "
"configured."
)
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index cce40fdd2d..7474fd515f 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -17,6 +17,7 @@
import logging
from collections import namedtuple
+from six import raise_from
from six.moves import urllib
from signedjson.key import (
@@ -35,7 +36,12 @@ from unpaddedbase64 import decode_base64
from twisted.internet import defer
-from synapse.api.errors import Codes, RequestSendFailed, SynapseError
+from synapse.api.errors import (
+ Codes,
+ HttpResponseException,
+ RequestSendFailed,
+ SynapseError,
+)
from synapse.util import logcontext, unwrapFirstError
from synapse.util.logcontext import (
LoggingContext,
@@ -44,6 +50,7 @@ from synapse.util.logcontext import (
run_in_background,
)
from synapse.util.metrics import Measure
+from synapse.util.retryutils import NotRetryingDestination
logger = logging.getLogger(__name__)
@@ -367,13 +374,18 @@ class Keyring(object):
server_name_and_key_ids, perspective_name, perspective_keys
)
defer.returnValue(result)
+ except KeyLookupError as e:
+ logger.warning(
+ "Key lookup failed from %r: %s", perspective_name, e,
+ )
except Exception as e:
logger.exception(
"Unable to get key from %r: %s %s",
perspective_name,
type(e).__name__, str(e),
)
- defer.returnValue({})
+
+ defer.returnValue({})
results = yield logcontext.make_deferred_yieldable(defer.gatherResults(
[
@@ -421,21 +433,30 @@ class Keyring(object):
# TODO(mark): Set the minimum_valid_until_ts to that needed by
# the events being validated or the current time if validating
# an incoming request.
- query_response = yield self.client.post_json(
- destination=perspective_name,
- path="/_matrix/key/v2/query",
- data={
- u"server_keys": {
- server_name: {
- key_id: {
- u"minimum_valid_until_ts": 0
- } for key_id in key_ids
+ try:
+ query_response = yield self.client.post_json(
+ destination=perspective_name,
+ path="/_matrix/key/v2/query",
+ data={
+ u"server_keys": {
+ server_name: {
+ key_id: {
+ u"minimum_valid_until_ts": 0
+ } for key_id in key_ids
+ }
+ for server_name, key_ids in server_names_and_key_ids
}
- for server_name, key_ids in server_names_and_key_ids
- }
- },
- long_retries=True,
- )
+ },
+ long_retries=True,
+ )
+ except (NotRetryingDestination, RequestSendFailed) as e:
+ raise_from(
+ KeyLookupError("Failed to connect to remote server"), e,
+ )
+ except HttpResponseException as e:
+ raise_from(
+ KeyLookupError("Remote server returned an error"), e,
+ )
keys = {}
@@ -502,11 +523,20 @@ class Keyring(object):
if requested_key_id in keys:
continue
- response = yield self.client.get_json(
- destination=server_name,
- path="/_matrix/key/v2/server/" + urllib.parse.quote(requested_key_id),
- ignore_backoff=True,
- )
+ try:
+ response = yield self.client.get_json(
+ destination=server_name,
+ path="/_matrix/key/v2/server/" + urllib.parse.quote(requested_key_id),
+ ignore_backoff=True,
+ )
+ except (NotRetryingDestination, RequestSendFailed) as e:
+ raise_from(
+ KeyLookupError("Failed to connect to remote server"), e,
+ )
+ except HttpResponseException as e:
+ raise_from(
+ KeyLookupError("Remote server returned an error"), e,
+ )
if (u"signatures" not in response
or server_name not in response[u"signatures"]):
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 4e4f58b418..58e04d81ab 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -33,6 +33,7 @@ from synapse.api.constants import (
)
from synapse.api.errors import (
CodeMessageException,
+ Codes,
FederationDeniedError,
HttpResponseException,
SynapseError,
@@ -792,10 +793,25 @@ class FederationClient(FederationBase):
defer.returnValue(content)
except HttpResponseException as e:
if e.code in [400, 404]:
+ err = e.to_synapse_error()
+
+ # If we receive an error response that isn't a generic error, we
+ # assume that the remote understands the v2 invite API and this
+ # is a legitimate error.
+ if err.errcode != Codes.UNKNOWN:
+ raise err
+
+ # Otherwise, we assume that the remote server doesn't understand
+ # the v2 invite API.
+
if room_version in (RoomVersions.V1, RoomVersions.V2):
pass # We'll fall through
else:
- raise Exception("Remote server is too old")
+ raise SynapseError(
+ 400,
+ "User's homeserver does not support this room version",
+ Codes.UNSUPPORTED_ROOM_VERSION,
+ )
elif e.code == 403:
raise e.to_synapse_error()
else:
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 3da86d4ba6..569eb277a9 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -25,9 +25,10 @@ from twisted.internet import defer
from twisted.internet.abstract import isIPAddress
from twisted.python import failure
-from synapse.api.constants import EventTypes, Membership
+from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership
from synapse.api.errors import (
AuthError,
+ Codes,
FederationError,
IncompatibleRoomVersionError,
NotFoundError,
@@ -239,8 +240,9 @@ class FederationServer(FederationBase):
f = failure.Failure()
pdu_results[event_id] = {"error": str(e)}
logger.error(
- "Failed to handle PDU %s: %s",
- event_id, f.getTraceback().rstrip(),
+ "Failed to handle PDU %s",
+ event_id,
+ exc_info=(f.type, f.value, f.getTracebackObject()),
)
yield concurrently_execute(
@@ -386,6 +388,13 @@ class FederationServer(FederationBase):
@defer.inlineCallbacks
def on_invite_request(self, origin, content, room_version):
+ if room_version not in KNOWN_ROOM_VERSIONS:
+ raise SynapseError(
+ 400,
+ "Homeserver does not support this room version",
+ Codes.UNSUPPORTED_ROOM_VERSION,
+ )
+
format_ver = room_version_to_event_format(room_version)
pdu = event_from_pdu_json(content, format_ver)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 083f2e0ac3..f80486102a 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -770,10 +770,26 @@ class FederationHandler(BaseHandler):
set(auth_events.keys()) | set(state_events.keys())
)
+ # We now have a chunk of events plus associated state and auth chain to
+ # persist. We do the persistence in two steps:
+ # 1. Auth events and state get persisted as outliers, plus the
+ # backward extremities get persisted (as non-outliers).
+ # 2. The rest of the events in the chunk get persisted one by one, as
+ # each one depends on the previous event for its state.
+ #
+ # The important thing is that events in the chunk get persisted as
+ # non-outliers, including when those events are also in the state or
+ # auth chain. Caution must therefore be taken to ensure that they are
+ # not accidentally marked as outliers.
+
+ # Step 1a: persist auth events that *don't* appear in the chunk
ev_infos = []
for a in auth_events.values():
- if a.event_id in seen_events:
+ # We only want to persist auth events as outliers that we haven't
+ # seen and aren't about to persist as part of the backfilled chunk.
+ if a.event_id in seen_events or a.event_id in event_map:
continue
+
a.internal_metadata.outlier = True
ev_infos.append({
"event": a,
@@ -785,14 +801,21 @@ class FederationHandler(BaseHandler):
}
})
+ # Step 1b: persist the events in the chunk we fetched state for (i.e.
+ # the backwards extremities) as non-outliers.
for e_id in events_to_state:
+ # For paranoia we ensure that these events are marked as
+ # non-outliers
+ ev = event_map[e_id]
+ assert(not ev.internal_metadata.is_outlier())
+
ev_infos.append({
- "event": event_map[e_id],
+ "event": ev,
"state": events_to_state[e_id],
"auth_events": {
(auth_events[a_id].type, auth_events[a_id].state_key):
auth_events[a_id]
- for a_id in event_map[e_id].auth_event_ids()
+ for a_id in ev.auth_event_ids()
if a_id in auth_events
}
})
@@ -802,12 +825,17 @@ class FederationHandler(BaseHandler):
backfilled=True,
)
+ # Step 2: Persist the rest of the events in the chunk one by one
events.sort(key=lambda e: e.depth)
for event in events:
if event in events_to_state:
continue
+ # For paranoia we ensure that these events are marked as
+ # non-outliers
+ assert(not event.internal_metadata.is_outlier())
+
# We store these one at a time since each event depends on the
# previous to work out the state.
# TODO: We can probably do something more clever here.
diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py
index 084c1503da..e4fdae9266 100644
--- a/synapse/handlers/pagination.py
+++ b/synapse/handlers/pagination.py
@@ -136,7 +136,11 @@ class PaginationHandler(object):
logger.info("[purge] complete")
self._purges_by_id[purge_id].status = PurgeStatus.STATUS_COMPLETE
except Exception:
- logger.error("[purge] failed: %s", Failure().getTraceback().rstrip())
+ f = Failure()
+ logger.error(
+ "[purge] failed",
+ exc_info=(f.type, f.value, f.getTracebackObject()),
+ )
self._purges_by_id[purge_id].status = PurgeStatus.STATUS_FAILED
finally:
self._purges_in_progress_by_room.discard(room_id)
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 24a4cb5a83..c0e06929bd 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -460,7 +460,7 @@ class RegistrationHandler(BaseHandler):
lines = response.split('\n')
json = {
"valid": lines[0] == 'true',
- "error_url": "http://www.google.com/recaptcha/api/challenge?" +
+ "error_url": "http://www.recaptcha.net/recaptcha/api/challenge?" +
"error=%s" % lines[1]
}
defer.returnValue(json)
@@ -471,7 +471,7 @@ class RegistrationHandler(BaseHandler):
Used only by c/s api v1
"""
data = yield self.captcha_client.post_urlencoded_get_raw(
- "http://www.google.com:80/recaptcha/api/verify",
+ "http://www.recaptcha.net:80/recaptcha/api/verify",
args={
'privatekey': private_key,
'remoteip': ip_addr,
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 6c67a25a11..16fb7935da 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -169,18 +169,18 @@ def _return_html_error(f, request):
)
else:
logger.error(
- "Failed handle request %r: %s",
+ "Failed handle request %r",
request,
- f.getTraceback().rstrip(),
+ exc_info=(f.type, f.value, f.getTracebackObject()),
)
else:
code = http_client.INTERNAL_SERVER_ERROR
msg = "Internal server error"
logger.error(
- "Failed handle request %r: %s",
+ "Failed handle request %r",
request,
- f.getTraceback().rstrip(),
+ exc_info=(f.type, f.value, f.getTracebackObject()),
)
body = HTML_ERROR_TEMPLATE.format(
diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py
index f7bb710642..ac035c7735 100644
--- a/synapse/rest/client/v2_alpha/auth.py
+++ b/synapse/rest/client/v2_alpha/auth.py
@@ -33,7 +33,7 @@ RECAPTCHA_TEMPLATE = """
<title>Authentication</title>
<meta name='viewport' content='width=device-width, initial-scale=1,
user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
-<script src="https://www.google.com/recaptcha/api.js"
+<script src="https://www.recaptcha.net/recaptcha/api.js"
async defer></script>
<script src="//code.jquery.com/jquery-1.11.2.min.js"></script>
<link rel="stylesheet" href="/_matrix/static/client/register/style.css">
diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index efe42a429d..d16a30acd8 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -133,8 +133,15 @@ def respond_with_responder(request, responder, media_type, file_size, upload_nam
logger.debug("Responding to media request with responder %s")
add_file_headers(request, media_type, file_size, upload_name)
- with responder:
- yield responder.write_to_consumer(request)
+ try:
+ with responder:
+ yield responder.write_to_consumer(request)
+ except Exception as e:
+ # The majority of the time this will be due to the client having gone
+ # away. Unfortunately, Twisted simply throws a generic exception at us
+ # in that case.
+ logger.warning("Failed to write to consumer: %s %s", type(e), e)
+
finish_request(request)
diff --git a/synapse/static/client/register/index.html b/synapse/static/client/register/index.html
index 886f2edd1f..6edc4deb03 100644
--- a/synapse/static/client/register/index.html
+++ b/synapse/static/client/register/index.html
@@ -4,7 +4,7 @@
<meta name='viewport' content='width=device-width, initial-scale=1, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
<link rel="stylesheet" href="style.css">
<script src="js/jquery-2.1.3.min.js"></script>
-<script src="https://www.google.com/recaptcha/api/js/recaptcha_ajax.js"></script>
+<script src="https://www.recaptcha.net/recaptcha/api/js/recaptcha_ajax.js"></script>
<script src="register_config.js"></script>
<script src="js/register.js"></script>
</head>
|
