From ae19a7db8c5eab43858b24453bbbb352f8b6152a Mon Sep 17 00:00:00 2001 From: rkfg Date: Thu, 6 Dec 2018 13:32:05 +0300 Subject: Prevent crash on pagination. --- synapse/handlers/pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 43f81bd607..f2be6c1185 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -253,7 +253,7 @@ class PaginationHandler(object): ) state = None - if event_filter and event_filter.lazy_load_members(): + if event_filter and event_filter.lazy_load_members() and len(events) > 0: # TODO: remove redundant members # FIXME: we also care about invite targets etc. -- cgit 1.5.1 From 6d65659b62d1e338987a071d2f053cc3447e7ff5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 21 Feb 2019 17:50:30 +0000 Subject: Run push_receipts_to_remotes as background job (#4707) I suspect the CPU usage metrics for this are going to /dev/null at the moment. --- changelog.d/4707.misc | 1 + synapse/handlers/receipts.py | 68 ++++++++++++++++++++++---------------------- 2 files changed, 35 insertions(+), 34 deletions(-) create mode 100644 changelog.d/4707.misc (limited to 'synapse') diff --git a/changelog.d/4707.misc b/changelog.d/4707.misc new file mode 100644 index 0000000000..ef0772b9af --- /dev/null +++ b/changelog.d/4707.misc @@ -0,0 +1 @@ +Run push_receipts_to_remotes as background job. diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 4c2690ba26..696469732c 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -16,8 +16,8 @@ import logging from twisted.internet import defer +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import get_domain_from_id -from synapse.util import logcontext from ._base import BaseHandler @@ -59,7 +59,9 @@ class ReceiptsHandler(BaseHandler): if is_new: # fire off a process in the background to send the receipt to # remote servers - self._push_remotes([receipt]) + run_as_background_process( + 'push_receipts_to_remotes', self._push_remotes, receipt + ) @defer.inlineCallbacks def _received_remote_receipt(self, origin, content): @@ -125,44 +127,42 @@ class ReceiptsHandler(BaseHandler): defer.returnValue(True) - @logcontext.preserve_fn # caller should not yield on this @defer.inlineCallbacks - def _push_remotes(self, receipts): - """Given a list of receipts, works out which remote servers should be + def _push_remotes(self, receipt): + """Given a receipt, works out which remote servers should be poked and pokes them. """ try: - # TODO: Some of this stuff should be coallesced. - for receipt in receipts: - room_id = receipt["room_id"] - receipt_type = receipt["receipt_type"] - user_id = receipt["user_id"] - event_ids = receipt["event_ids"] - data = receipt["data"] - - users = yield self.state.get_current_user_in_room(room_id) - remotedomains = set(get_domain_from_id(u) for u in users) - remotedomains = remotedomains.copy() - remotedomains.discard(self.server_name) - - logger.debug("Sending receipt to: %r", remotedomains) - - for domain in remotedomains: - self.federation.send_edu( - destination=domain, - edu_type="m.receipt", - content={ - room_id: { - receipt_type: { - user_id: { - "event_ids": event_ids, - "data": data, - } + # TODO: optimise this to move some of the work to the workers. + room_id = receipt["room_id"] + receipt_type = receipt["receipt_type"] + user_id = receipt["user_id"] + event_ids = receipt["event_ids"] + data = receipt["data"] + + users = yield self.state.get_current_user_in_room(room_id) + remotedomains = set(get_domain_from_id(u) for u in users) + remotedomains = remotedomains.copy() + remotedomains.discard(self.server_name) + + logger.debug("Sending receipt to: %r", remotedomains) + + for domain in remotedomains: + self.federation.send_edu( + destination=domain, + edu_type="m.receipt", + content={ + room_id: { + receipt_type: { + user_id: { + "event_ids": event_ids, + "data": data, } - }, + } }, - key=(room_id, receipt_type, user_id), - ) + }, + key=(room_id, receipt_type, user_id), + ) except Exception: logger.exception("Error pushing receipts to remote servers") -- cgit 1.5.1 From 0abb094f1ada9c4a78a11eb06b205c00db826860 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 21 Feb 2019 17:51:21 +0000 Subject: bail out early in on_new_receipts if no pushers (#4706) --- changelog.d/4706.misc | 1 + synapse/push/pusherpool.py | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog.d/4706.misc (limited to 'synapse') diff --git a/changelog.d/4706.misc b/changelog.d/4706.misc new file mode 100644 index 0000000000..73d1ddcc56 --- /dev/null +++ b/changelog.d/4706.misc @@ -0,0 +1 @@ +Avoid some redundant work when processing read receipts diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 5a4e73ccd6..b2c92ce683 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -140,6 +140,10 @@ class PusherPool: @defer.inlineCallbacks def on_new_notifications(self, min_stream_id, max_stream_id): + if not self.pushers: + # nothing to do here. + return + try: users_affected = yield self.store.get_push_action_users_in_range( min_stream_id, max_stream_id @@ -155,6 +159,10 @@ class PusherPool: @defer.inlineCallbacks def on_new_receipts(self, min_stream_id, max_stream_id, affected_room_ids): + if not self.pushers: + # nothing to do here. + return + try: # Need to subtract 1 from the minimum because the lower bound here # is not inclusive -- cgit 1.5.1 From e07384c4e16a8a97966c58d604ef95ac1614cafa Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 22 Feb 2019 10:57:15 +0000 Subject: Add prometheus metrics for number of badge update pushes. (#4709) We're counting the number of push notifications, but not the number of badges; I'd like to see if they are significant. --- changelog.d/4709.misc | 1 + synapse/push/httppusher.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 changelog.d/4709.misc (limited to 'synapse') diff --git a/changelog.d/4709.misc b/changelog.d/4709.misc new file mode 100644 index 0000000000..ca47a6f327 --- /dev/null +++ b/changelog.d/4709.misc @@ -0,0 +1 @@ +Add prometheus metrics for number of badge update pushes. diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 98d8d9560b..899a5253d8 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -32,9 +32,25 @@ if six.PY3: logger = logging.getLogger(__name__) -http_push_processed_counter = Counter("synapse_http_httppusher_http_pushes_processed", "") +http_push_processed_counter = Counter( + "synapse_http_httppusher_http_pushes_processed", + "Number of push notifications successfully sent", +) -http_push_failed_counter = Counter("synapse_http_httppusher_http_pushes_failed", "") +http_push_failed_counter = Counter( + "synapse_http_httppusher_http_pushes_failed", + "Number of push notifications which failed", +) + +http_badges_processed_counter = Counter( + "synapse_http_httppusher_badge_updates_processed", + "Number of badge updates successfully sent", +) + +http_badges_failed_counter = Counter( + "synapse_http_httppusher_badge_updates_failed", + "Number of badge updates which failed", +) class HttpPusher(object): @@ -346,6 +362,10 @@ class HttpPusher(object): @defer.inlineCallbacks def _send_badge(self, badge): + """ + Args: + badge (int): number of unread messages + """ logger.info("Sending updated badge count %d to %s", badge, self.name) d = { 'notification': { @@ -366,14 +386,11 @@ class HttpPusher(object): } } try: - resp = yield self.http_client.post_json_get_json(self.url, d) + yield self.http_client.post_json_get_json(self.url, d) + http_badges_processed_counter.inc() except Exception as e: logger.warning( "Failed to send badge count to %s: %s %s", self.name, type(e), e, ) - defer.returnValue(False) - rejected = [] - if 'rejected' in resp: - rejected = resp['rejected'] - defer.returnValue(rejected) + http_badges_failed_counter.inc() -- cgit 1.5.1 From 7b288826b76cae63b553fced10d3779355b592ca Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Feb 2019 11:33:51 +0000 Subject: Fix backfill storing incorrect state for events --- synapse/handlers/federation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 083f2e0ac3..40655bf92c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -772,8 +772,11 @@ class FederationHandler(BaseHandler): ev_infos = [] for a in auth_events.values(): - if a.event_id in seen_events: + # We only want to persist auth events as outliers that we haven't + # seen and aren't about to persist as part of the backfilled chunk. + if a.event_id in seen_events or a.event_id in event_map: continue + a.internal_metadata.outlier = True ev_infos.append({ "event": a, -- cgit 1.5.1 From 80467bbac3be6e008b807793dfd27c733936c15c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Feb 2019 14:38:14 +0000 Subject: Fix state cache invalidation on workers --- synapse/replication/slave/storage/_base.py | 7 +----- synapse/storage/_base.py | 40 +++++++++++++++++++++++++----- 2 files changed, 35 insertions(+), 12 deletions(-) (limited to 'synapse') diff --git a/synapse/replication/slave/storage/_base.py b/synapse/replication/slave/storage/_base.py index 1353a32d00..817d1f67f9 100644 --- a/synapse/replication/slave/storage/_base.py +++ b/synapse/replication/slave/storage/_base.py @@ -59,12 +59,7 @@ class BaseSlavedStore(SQLBaseStore): members_changed = set(row.keys[1:]) self._invalidate_state_caches(room_id, members_changed) else: - try: - getattr(self, row.cache_func).invalidate(tuple(row.keys)) - except AttributeError: - # We probably haven't pulled in the cache in this worker, - # which is fine. - pass + self._attempt_to_invalidate_cache(row.cache_func, tuple(row.keys)) def _invalidate_cache_and_stream(self, txn, cache_func, keys): txn.call_after(cache_func.invalidate, keys) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 3d895da43c..5a80eef211 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -1342,15 +1342,43 @@ class SQLBaseStore(object): changed """ for member in members_changed: - self.get_rooms_for_user_with_stream_ordering.invalidate((member,)) + self._attempt_to_invalidate_cache( + "get_rooms_for_user_with_stream_ordering", (member,), + ) for host in set(get_domain_from_id(u) for u in members_changed): - self.is_host_joined.invalidate((room_id, host)) - self.was_host_joined.invalidate((room_id, host)) + self._attempt_to_invalidate_cache( + "is_host_joined", (room_id, host,), + ) + self._attempt_to_invalidate_cache( + "was_host_joined", (room_id, host,), + ) + + self._attempt_to_invalidate_cache( + "get_users_in_room", (room_id,), + ) + self._attempt_to_invalidate_cache( + "get_room_summary", (room_id,), + ) + self._attempt_to_invalidate_cache( + "get_current_state_ids", (room_id,), + ) + + def _attempt_to_invalidate_cache(self, cache_name, key): + """Attempts to invalidate the cache of the given name, ignoring if the + cache doesn't exist. Mainly used for invalidating caches on workers, + where they may not have the cache. - self.get_users_in_room.invalidate((room_id,)) - self.get_room_summary.invalidate((room_id,)) - self.get_current_state_ids.invalidate((room_id,)) + Args: + cache_name (str) + key (tuple) + """ + try: + getattr(self, cache_name).invalidate(key) + except AttributeError: + # We probably haven't pulled in the cache in this worker, + # which is fine. + pass def _send_invalidation_to_replication(self, txn, cache_name, keys): """Notifies replication that given cache has been invalidated. -- cgit 1.5.1 From 1d9df51ff1129f86157e54f0523cec0ddadcbd41 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Feb 2019 14:47:48 +0000 Subject: Correctly handle null data in HttpPusher --- synapse/push/httppusher.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'synapse') diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 899a5253d8..e65f8c63d3 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -97,6 +97,11 @@ class HttpPusher(object): pusherdict['pushkey'], ) + if self.data is None: + raise PusherConfigException( + "data can not be null for HTTP pusher" + ) + if 'url' not in self.data: raise PusherConfigException( "'url' required in data for HTTP pusher" -- cgit 1.5.1 From a164134a53417bfa3f126671b02be7e83376aacd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Feb 2019 14:48:06 +0000 Subject: Drop logging level of creating a pusher --- synapse/push/pusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py index 368d5094be..b33f2a357b 100644 --- a/synapse/push/pusher.py +++ b/synapse/push/pusher.py @@ -56,7 +56,7 @@ class PusherFactory(object): f = self.pusher_types.get(kind, None) if not f: return None - logger.info("creating %s pusher for %r", kind, pusherdict) + logger.debug("creating %s pusher for %r", kind, pusherdict) return f(self.hs, pusherdict) def _create_email_pusher(self, _hs, pusherdict): -- cgit 1.5.1 From f2891d248756c968c346df052c796c0c69d6e764 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Feb 2019 15:18:19 +0000 Subject: Correctly handle PusherConfigException --- synapse/push/pusherpool.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'synapse') diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index b2c92ce683..af82e02b46 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -19,6 +19,7 @@ import logging from twisted.internet import defer from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.push import PusherConfigException from synapse.push.pusher import PusherFactory logger = logging.getLogger(__name__) @@ -222,6 +223,14 @@ class PusherPool: """ try: p = self.pusher_factory.create_pusher(pusherdict) + except PusherConfigException as e: + logger.warning( + "Pusher incorrectly configured user=%s, appid=%s, pushkey=%s: %s", + pusherdict.get('user_name'), + pusherdict.get('app_id'), + pusherdict.get('pushkey'), + e, + ) except Exception: logger.exception("Couldn't start a pusher: caught Exception") return -- cgit 1.5.1 From b82c9cf4629040681c7d06846422705734acd110 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Feb 2019 15:27:40 +0000 Subject: Add missing return --- synapse/push/pusherpool.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index af82e02b46..abf1a1a9c1 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -231,6 +231,7 @@ class PusherPool: pusherdict.get('pushkey'), e, ) + return except Exception: logger.exception("Couldn't start a pusher: caught Exception") return -- cgit 1.5.1 From 59e011220942e642968c453a36fff4862c753ce3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 23 Feb 2019 14:31:08 +0000 Subject: MSC 1866 - Use M_UNSUPPORTED_ROOM_VERSION for invite API --- synapse/federation/federation_client.py | 18 +++++++++++++++++- synapse/federation/federation_server.py | 10 +++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 4e4f58b418..58e04d81ab 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -33,6 +33,7 @@ from synapse.api.constants import ( ) from synapse.api.errors import ( CodeMessageException, + Codes, FederationDeniedError, HttpResponseException, SynapseError, @@ -792,10 +793,25 @@ class FederationClient(FederationBase): defer.returnValue(content) except HttpResponseException as e: if e.code in [400, 404]: + err = e.to_synapse_error() + + # If we receive an error response that isn't a generic error, we + # assume that the remote understands the v2 invite API and this + # is a legitimate error. + if err.errcode != Codes.UNKNOWN: + raise err + + # Otherwise, we assume that the remote server doesn't understand + # the v2 invite API. + if room_version in (RoomVersions.V1, RoomVersions.V2): pass # We'll fall through else: - raise Exception("Remote server is too old") + raise SynapseError( + 400, + "User's homeserver does not support this room version", + Codes.UNSUPPORTED_ROOM_VERSION, + ) elif e.code == 403: raise e.to_synapse_error() else: diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 3da86d4ba6..e44342bc85 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -25,9 +25,10 @@ from twisted.internet import defer from twisted.internet.abstract import isIPAddress from twisted.python import failure -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership from synapse.api.errors import ( AuthError, + Codes, FederationError, IncompatibleRoomVersionError, NotFoundError, @@ -386,6 +387,13 @@ class FederationServer(FederationBase): @defer.inlineCallbacks def on_invite_request(self, origin, content, room_version): + if room_version not in KNOWN_ROOM_VERSIONS: + raise SynapseError( + 400, + "Homeserver does not support this room version", + Codes.UNSUPPORTED_ROOM_VERSION, + ) + format_ver = room_version_to_event_format(room_version) pdu = event_from_pdu_json(content, format_ver) -- cgit 1.5.1 From 41285ffe5b4ebb57e1296f8ace903a2e97a81f7d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 23 Feb 2019 15:06:02 +0000 Subject: Handle errors when fetching remote server keys --- synapse/crypto/keyring.py | 72 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 21 deletions(-) (limited to 'synapse') diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index cce40fdd2d..8c125e3a46 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -17,6 +17,7 @@ import logging from collections import namedtuple +from six import raise_from from six.moves import urllib from signedjson.key import ( @@ -35,7 +36,12 @@ from unpaddedbase64 import decode_base64 from twisted.internet import defer -from synapse.api.errors import Codes, RequestSendFailed, SynapseError +from synapse.api.errors import ( + Codes, + HttpResponseException, + RequestSendFailed, + SynapseError, +) from synapse.util import logcontext, unwrapFirstError from synapse.util.logcontext import ( LoggingContext, @@ -44,6 +50,7 @@ from synapse.util.logcontext import ( run_in_background, ) from synapse.util.metrics import Measure +from synapse.util.retryutils import NotRetryingDestination logger = logging.getLogger(__name__) @@ -367,13 +374,18 @@ class Keyring(object): server_name_and_key_ids, perspective_name, perspective_keys ) defer.returnValue(result) + except KeyLookupError as e: + logger.warning( + "Key lookup failed from %r: %s", perspective_name, e, + ) except Exception as e: logger.exception( "Unable to get key from %r: %s %s", perspective_name, type(e).__name__, str(e), ) - defer.returnValue({}) + + defer.returnValue({}) results = yield logcontext.make_deferred_yieldable(defer.gatherResults( [ @@ -421,21 +433,30 @@ class Keyring(object): # TODO(mark): Set the minimum_valid_until_ts to that needed by # the events being validated or the current time if validating # an incoming request. - query_response = yield self.client.post_json( - destination=perspective_name, - path="/_matrix/key/v2/query", - data={ - u"server_keys": { - server_name: { - key_id: { - u"minimum_valid_until_ts": 0 - } for key_id in key_ids + try: + query_response = yield self.client.post_json( + destination=perspective_name, + path="/_matrix/key/v2/query", + data={ + u"server_keys": { + server_name: { + key_id: { + u"minimum_valid_until_ts": 0 + } for key_id in key_ids + } + for server_name, key_ids in server_names_and_key_ids } - for server_name, key_ids in server_names_and_key_ids - } - }, - long_retries=True, - ) + }, + long_retries=True, + ) + except (NotRetryingDestination, RequestSendFailed) as e: + raise raise_from( + KeyLookupError("Failed to connect to remote server"), e, + ) + except HttpResponseException as e: + raise raise_from( + KeyLookupError("Remote server returned an error"), e, + ) keys = {} @@ -502,11 +523,20 @@ class Keyring(object): if requested_key_id in keys: continue - response = yield self.client.get_json( - destination=server_name, - path="/_matrix/key/v2/server/" + urllib.parse.quote(requested_key_id), - ignore_backoff=True, - ) + try: + response = yield self.client.get_json( + destination=server_name, + path="/_matrix/key/v2/server/" + urllib.parse.quote(requested_key_id), + ignore_backoff=True, + ) + except (NotRetryingDestination, RequestSendFailed) as e: + raise raise_from( + KeyLookupError("Failed to connect to remote server"), e, + ) + except HttpResponseException as e: + raise raise_from( + KeyLookupError("Remote server returned an error"), e, + ) if (u"signatures" not in response or server_name not in response[u"signatures"]): -- cgit 1.5.1 From 47a7e3928d103b88ede86fb742cd05d092f5ee62 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 23 Feb 2019 15:17:57 +0000 Subject: Correctly proxy exception in frontend_proxy worker --- synapse/app/frontend_proxy.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index d5b954361d..8479fee738 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -21,7 +21,7 @@ from twisted.web.resource import NoResource import synapse from synapse import events -from synapse.api.errors import SynapseError +from synapse.api.errors import HttpResponseException, SynapseError from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig @@ -66,10 +66,15 @@ class PresenceStatusStubServlet(ClientV1RestServlet): headers = { "Authorization": auth_headers, } - result = yield self.http_client.get_json( - self.main_uri + request.uri.decode('ascii'), - headers=headers, - ) + + try: + result = yield self.http_client.get_json( + self.main_uri + request.uri.decode('ascii'), + headers=headers, + ) + except HttpResponseException as e: + raise e.to_synapse_error() + defer.returnValue((200, result)) @defer.inlineCallbacks -- cgit 1.5.1 From 9342cc6ab167836ca3965923989c33f5230c27e9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 10:02:12 +0000 Subject: Add comments and paranoia --- synapse/handlers/federation.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 40655bf92c..fbf044b407 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -770,6 +770,18 @@ class FederationHandler(BaseHandler): set(auth_events.keys()) | set(state_events.keys()) ) + # We now have a chunk of events plus associated state and auth chain to + # persist. We do the persistence in two steps: + # 1. Auth events and state get persisted as outliers, plus the + # backward extremities get persisted (as non-outliers). + # 2. The rest of the events in the chunk get persisted one by one, as + # each one depends on the previous event for its state. + # + # The important thing is that events in the chunk get persisted as + # non-outliers, including when those events are also in the state or + # auth chain. Caution must therefore be taken to ensure that they are + # not accidentally marked as outliers. + ev_infos = [] for a in auth_events.values(): # We only want to persist auth events as outliers that we haven't @@ -789,13 +801,18 @@ class FederationHandler(BaseHandler): }) for e_id in events_to_state: + # For paranoia we ensure that these events are marked as + # non-outliers + ev = event_map[e_id] + ev.internal_metadata.outlier = False + ev_infos.append({ - "event": event_map[e_id], + "event": ev, "state": events_to_state[e_id], "auth_events": { (auth_events[a_id].type, auth_events[a_id].state_key): auth_events[a_id] - for a_id in event_map[e_id].auth_event_ids() + for a_id in ev.auth_event_ids() if a_id in auth_events } }) @@ -811,6 +828,10 @@ class FederationHandler(BaseHandler): if event in events_to_state: continue + # For paranoia we ensure that these events are marked as + # non-outliers + event.internal_metadata.outlier = False + # We store these one at a time since each event depends on the # previous to work out the state. # TODO: We can probably do something more clever here. -- cgit 1.5.1 From 65d1003d01f7dc301340224be70e9999c2c50065 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 14:34:03 +0000 Subject: raise_from already raises --- synapse/crypto/keyring.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 8c125e3a46..7474fd515f 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -450,11 +450,11 @@ class Keyring(object): long_retries=True, ) except (NotRetryingDestination, RequestSendFailed) as e: - raise raise_from( + raise_from( KeyLookupError("Failed to connect to remote server"), e, ) except HttpResponseException as e: - raise raise_from( + raise_from( KeyLookupError("Remote server returned an error"), e, ) @@ -530,11 +530,11 @@ class Keyring(object): ignore_backoff=True, ) except (NotRetryingDestination, RequestSendFailed) as e: - raise raise_from( + raise_from( KeyLookupError("Failed to connect to remote server"), e, ) except HttpResponseException as e: - raise raise_from( + raise_from( KeyLookupError("Remote server returned an error"), e, ) -- cgit 1.5.1 From 890cb048fdbb05bcd8b80d3038f4811a1fdfe9f0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 14:42:11 +0000 Subject: Assert rather than clobber the values --- synapse/handlers/federation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index fbf044b407..8e6d0a3bbc 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -804,7 +804,7 @@ class FederationHandler(BaseHandler): # For paranoia we ensure that these events are marked as # non-outliers ev = event_map[e_id] - ev.internal_metadata.outlier = False + assert(not event.internal_metadata.is_outlier()) ev_infos.append({ "event": ev, @@ -830,7 +830,7 @@ class FederationHandler(BaseHandler): # For paranoia we ensure that these events are marked as # non-outliers - event.internal_metadata.outlier = False + assert(not event.internal_metadata.is_outlier()) # We store these one at a time since each event depends on the # previous to work out the state. -- cgit 1.5.1 From d730c2c22b53bef52afb6b26a878acc186e4ecae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 14:45:02 +0000 Subject: More comments --- synapse/handlers/federation.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8e6d0a3bbc..ca9b281be1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -782,6 +782,7 @@ class FederationHandler(BaseHandler): # auth chain. Caution must therefore be taken to ensure that they are # not accidentally marked as outliers. + # Step 1a: persist auth events that *don't* appear in the chunk ev_infos = [] for a in auth_events.values(): # We only want to persist auth events as outliers that we haven't @@ -800,6 +801,8 @@ class FederationHandler(BaseHandler): } }) + # Step 1b: persist the events in the chunk we fetched state for (i.e. + # the backwards extremities) as non-outliers. for e_id in events_to_state: # For paranoia we ensure that these events are marked as # non-outliers @@ -822,6 +825,7 @@ class FederationHandler(BaseHandler): backfilled=True, ) + # Step 2: Persist the rest of the events in the chunk one by one events.sort(key=lambda e: e.depth) for event in events: -- cgit 1.5.1 From e4b078a6004ea34684c27056825346178c22aa0f Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 22 Feb 2019 12:11:43 +0000 Subject: Config option to prevent showing non-fed rooms in fed /publicRooms --- synapse/config/room_directory.py | 11 +++++++++++ synapse/handlers/room_list.py | 17 +++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py index 9da13ab11b..08d1d26e58 100644 --- a/synapse/config/room_directory.py +++ b/synapse/config/room_directory.py @@ -27,6 +27,10 @@ class RoomDirectoryConfig(Config): for rule in alias_creation_rules ] + self.allow_non_federated_in_public_rooms = config.get( + "allow_non_federated_in_public_rooms", True, + ) + def default_config(self, config_dir_path, server_name, **kwargs): return """ # The `alias_creation` option controls who's allowed to create aliases @@ -42,6 +46,13 @@ class RoomDirectoryConfig(Config): - user_id: "*" alias: "*" action: allow + + # Specify whether rooms that only allow local users to join should be + # shown in the federation public room directory. + # + # Note that this does not affect the room directory shown to users on + # this homeserver, only those on other homeservers. + #allow_non_federated_in_public_rooms: True """ def is_alias_creation_allowed(self, user_id, alias): diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index dc88620885..47aeeb1d88 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -47,6 +47,7 @@ class RoomListHandler(BaseHandler): self.response_cache = ResponseCache(hs, "room_list") self.remote_response_cache = ResponseCache(hs, "remote_room_list", timeout_ms=30 * 1000) + self.config = hs.get_config() def get_local_public_room_list(self, limit=None, since_token=None, search_filter=None, @@ -286,14 +287,16 @@ class RoomListHandler(BaseHandler): # We've already got enough, so lets just drop it. return - result = yield self.generate_room_entry(room_id, num_joined_users) + result = yield self.generate_room_entry(room_id, num_joined_users, + allow_federated=self.config.allow_non_federated_in_public_rooms) if result and _matches_room_entry(result, search_filter): chunk.append(result) @cachedInlineCallbacks(num_args=1, cache_context=True) def generate_room_entry(self, room_id, num_joined_users, cache_context, - with_alias=True, allow_private=False): + with_alias=True, allow_private=False, + allow_federated=True): """Returns the entry for a room """ result = { @@ -308,6 +311,7 @@ class RoomListHandler(BaseHandler): event_map = yield self.store.get_events([ event_id for key, event_id in iteritems(current_state_ids) if key[0] in ( + EventTypes.Create, EventTypes.JoinRules, EventTypes.Name, EventTypes.Topic, @@ -324,12 +328,21 @@ class RoomListHandler(BaseHandler): } # Double check that this is actually a public room. + join_rules_event = current_state.get((EventTypes.JoinRules, "")) if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) if not allow_private and join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) + if not allow_federated: + # Disallow non-federated from appearing + create_event = current_state.get((EventTypes.Create, "")) + if create_event: + federate = create_event.content.get("m.federate", True) + if federate == False: + defer.returnValue(None) + if with_alias: aliases = yield self.store.get_aliases_for_room( room_id, on_invalidate=cache_context.invalidate -- cgit 1.5.1 From bd398b874eb375e92939a087a64258a4feb91f49 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 22 Feb 2019 15:43:11 +0000 Subject: Don't restrict non-fed rooms over client APIs --- synapse/federation/transport/server.py | 3 ++- synapse/groups/groups_server.py | 4 ++-- synapse/handlers/room_list.py | 29 ++++++++++++++++++----------- 3 files changed, 22 insertions(+), 14 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 7288d49074..6d4a26f595 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -697,7 +697,8 @@ class PublicRoomList(BaseFederationServlet): data = yield self.handler.get_local_public_room_list( limit, since_token, - network_tuple=network_tuple + network_tuple=network_tuple, + from_federation=True, ) defer.returnValue((200, data)) diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index 633c865ed8..691752a30c 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -113,7 +113,7 @@ class GroupsServerHandler(object): room_id = room_entry["room_id"] joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), + room_id, True, len(joined_users), with_alias=False, allow_private=True, ) entry = dict(entry) # so we don't change whats cached @@ -544,7 +544,7 @@ class GroupsServerHandler(object): joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), + room_id, True, len(joined_users), with_alias=False, allow_private=True, ) diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 47aeeb1d88..bde9ec744f 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -51,7 +51,8 @@ class RoomListHandler(BaseHandler): def get_local_public_room_list(self, limit=None, since_token=None, search_filter=None, - network_tuple=EMPTY_THIRD_PARTY_ID,): + network_tuple=EMPTY_THIRD_PARTY_ID, + from_federation=False): """Generate a local public room list. There are multiple different lists: the main one plus one per third @@ -82,13 +83,15 @@ class RoomListHandler(BaseHandler): return self.response_cache.wrap( key, self._get_public_room_list, - limit, since_token, network_tuple=network_tuple, + limit, since_token, + network_tuple=network_tuple, from_federation=from_federation, ) @defer.inlineCallbacks def _get_public_room_list(self, limit=None, since_token=None, search_filter=None, - network_tuple=EMPTY_THIRD_PARTY_ID,): + network_tuple=EMPTY_THIRD_PARTY_ID, + from_federation=False,): if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: @@ -208,7 +211,8 @@ class RoomListHandler(BaseHandler): yield concurrently_execute( lambda r: self._append_room_entry_to_chunk( r, rooms_to_num_joined[r], - chunk, limit, search_filter + chunk, limit, search_filter, + from_federation=from_federation, ), batch, 5, ) @@ -279,7 +283,7 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def _append_room_entry_to_chunk(self, room_id, num_joined_users, chunk, limit, - search_filter): + search_filter, from_federation=False): """Generate the entry for a room in the public room list and append it to the `chunk` if it matches the search filter """ @@ -287,16 +291,19 @@ class RoomListHandler(BaseHandler): # We've already got enough, so lets just drop it. return - result = yield self.generate_room_entry(room_id, num_joined_users, - allow_federated=self.config.allow_non_federated_in_public_rooms) + if from_federation: + result = yield self.generate_room_entry(room_id, + self.config.allow_non_federated_in_public_rooms, + num_joined_users) + else: + result = yield self.generate_room_entry(room_id, True, num_joined_users) if result and _matches_room_entry(result, search_filter): chunk.append(result) - @cachedInlineCallbacks(num_args=1, cache_context=True) - def generate_room_entry(self, room_id, num_joined_users, cache_context, - with_alias=True, allow_private=False, - allow_federated=True): + @cachedInlineCallbacks(num_args=2, cache_context=True) + def generate_room_entry(self, room_id, allow_federated, num_joined_users, + cache_context, with_alias=True, allow_private=False): """Returns the entry for a room """ result = { -- cgit 1.5.1 From 07493607a84665be006ec52f897eb904d923721c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 25 Feb 2019 12:42:30 +0000 Subject: Docs and arg name clarification --- synapse/handlers/room_list.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index bde9ec744f..0636e5b8eb 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -302,9 +302,22 @@ class RoomListHandler(BaseHandler): chunk.append(result) @cachedInlineCallbacks(num_args=2, cache_context=True) - def generate_room_entry(self, room_id, allow_federated, num_joined_users, + def generate_room_entry(self, room_id, allow_non_federated, num_joined_users, cache_context, with_alias=True, allow_private=False): """Returns the entry for a room + + Args: + room_id (str): The room's ID. + allow_non_federated (bool): Whether rooms with federation + disabled should be shown. + num_joined_users (int): Number of users in the room. + cache_context: Information for cached responses. + with_alias (bool): Whether to return the room's aliases in the result. + allow_private (bool): Whether invite-only rooms should be shown. + + Returns: + Deferred[dict|None]: Returns a room entry as a dictionary, or None if this + room was determined not to be shown publicly. """ result = { "room_id": room_id, @@ -342,7 +355,7 @@ class RoomListHandler(BaseHandler): if not allow_private and join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) - if not allow_federated: + if not allow_non_federated: # Disallow non-federated from appearing create_event = current_state.get((EventTypes.Create, "")) if create_event: -- cgit 1.5.1 From 84c0a20dfeb043f1113a6e213bc7488ef21a0432 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 25 Feb 2019 15:23:27 +0000 Subject: Simplify call to generate_room_entry --- synapse/handlers/room_list.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 810c624c2b..69b0bc3dd1 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -300,24 +300,24 @@ class RoomListHandler(BaseHandler): # We've already got enough, so lets just drop it. return - if from_federation: - result = yield self.generate_room_entry(room_id, - self.config.allow_non_federated_in_public_rooms, - num_joined_users) - else: - result = yield self.generate_room_entry(room_id, True, num_joined_users) - - if result and _matches_room_entry(result, search_filter): + result = yield self.generate_room_entry(room_id, + num_joined_users) + + if from_federation and not self.config.allow_non_federated_in_public_rooms: + if result["m.federate"] = False: + # This is a non-federating room and the config has chosen not + # to show these rooms to other servers + chunk.append(None) + else if result and _matches_room_entry(result, search_filter): chunk.append(result) @cachedInlineCallbacks(num_args=2, cache_context=True) - def generate_room_entry(self, room_id, allow_non_federated, num_joined_users, + def generate_room_entry(self, room_id, num_joined_users, cache_context, with_alias=True, allow_private=False): """Returns the entry for a room Args: room_id (str): The room's ID. - allow_non_federated (bool): Whether rooms with federation disabled should be shown. num_joined_users (int): Number of users in the room. cache_context: Information for cached responses. @@ -364,13 +364,9 @@ class RoomListHandler(BaseHandler): if not allow_private and join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) - if not allow_non_federated: - # Disallow non-federated from appearing - create_event = current_state.get((EventTypes.Create, "")) - if create_event: - federate = create_event.content.get("m.federate", True) - if federate == False: - defer.returnValue(None) + # Return whether this room is open to federation users or not + create_event = current_state.get((EventTypes.Create, "")) + result["m.federate"] = create_event.content.get("m.federate", True) if with_alias: aliases = yield self.store.get_aliases_for_room( -- cgit 1.5.1 From 8aaf7ffc4415e9f7da2d34448b29fb2d0cb80574 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 25 Feb 2019 15:27:17 +0000 Subject: syntax derp --- synapse/handlers/room_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 69b0bc3dd1..b7f450e719 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -304,7 +304,7 @@ class RoomListHandler(BaseHandler): num_joined_users) if from_federation and not self.config.allow_non_federated_in_public_rooms: - if result["m.federate"] = False: + if result["m.federate"] == False: # This is a non-federating room and the config has chosen not # to show these rooms to other servers chunk.append(None) -- cgit 1.5.1 From 9c598dddcbd75f60d422164215f3b078def604b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 16:32:02 +0000 Subject: Fix typo --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ca9b281be1..f80486102a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -807,7 +807,7 @@ class FederationHandler(BaseHandler): # For paranoia we ensure that these events are marked as # non-outliers ev = event_map[e_id] - assert(not event.internal_metadata.is_outlier()) + assert(not ev.internal_metadata.is_outlier()) ev_infos.append({ "event": ev, -- cgit 1.5.1 From c7b333c545adf8958ec97771e6816ceb5e83f524 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 16:56:41 +0000 Subject: Log tracebacks correctly --- synapse/federation/federation_server.py | 5 +++-- synapse/handlers/pagination.py | 6 +++++- synapse/http/server.py | 8 ++++---- 3 files changed, 12 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 3da86d4ba6..8f45c2badf 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -239,8 +239,9 @@ class FederationServer(FederationBase): f = failure.Failure() pdu_results[event_id] = {"error": str(e)} logger.error( - "Failed to handle PDU %s: %s", - event_id, f.getTraceback().rstrip(), + "Failed to handle PDU %s", + event_id, + exc_info=(f.type, f.value, f.getTracebackObject()), ) yield concurrently_execute( diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 084c1503da..e4fdae9266 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -136,7 +136,11 @@ class PaginationHandler(object): logger.info("[purge] complete") self._purges_by_id[purge_id].status = PurgeStatus.STATUS_COMPLETE except Exception: - logger.error("[purge] failed: %s", Failure().getTraceback().rstrip()) + f = Failure() + logger.error( + "[purge] failed", + exc_info=(f.type, f.value, f.getTracebackObject()), + ) self._purges_by_id[purge_id].status = PurgeStatus.STATUS_FAILED finally: self._purges_in_progress_by_room.discard(room_id) diff --git a/synapse/http/server.py b/synapse/http/server.py index 6c67a25a11..16fb7935da 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -169,18 +169,18 @@ def _return_html_error(f, request): ) else: logger.error( - "Failed handle request %r: %s", + "Failed handle request %r", request, - f.getTraceback().rstrip(), + exc_info=(f.type, f.value, f.getTracebackObject()), ) else: code = http_client.INTERNAL_SERVER_ERROR msg = "Internal server error" logger.error( - "Failed handle request %r: %s", + "Failed handle request %r", request, - f.getTraceback().rstrip(), + exc_info=(f.type, f.value, f.getTracebackObject()), ) body = HTML_ERROR_TEMPLATE.format( -- cgit 1.5.1 From 1330aa4a8fdff5f6d1ff4bc5d27de674dfcd67e7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 25 Feb 2019 17:28:19 +0000 Subject: elif not else if --- synapse/handlers/room_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index b7f450e719..94cabc2a2e 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -308,7 +308,7 @@ class RoomListHandler(BaseHandler): # This is a non-federating room and the config has chosen not # to show these rooms to other servers chunk.append(None) - else if result and _matches_room_entry(result, search_filter): + elif result and _matches_room_entry(result, search_filter): chunk.append(result) @cachedInlineCallbacks(num_args=2, cache_context=True) -- cgit 1.5.1 From 96c408273e3ed8fd18ca3f7530820a02c06c42c1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 25 Feb 2019 18:00:17 +0000 Subject: Fix group's call to generate_room_entry --- synapse/groups/groups_server.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index 691752a30c..a7eaead56b 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -113,8 +113,7 @@ class GroupsServerHandler(object): room_id = room_entry["room_id"] joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, True, len(joined_users), - with_alias=False, allow_private=True, + room_id, len(joined_users), with_alias=False, allow_private=True, ) entry = dict(entry) # so we don't change whats cached entry.pop("room_id", None) @@ -544,8 +543,7 @@ class GroupsServerHandler(object): joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, True, len(joined_users), - with_alias=False, allow_private=True, + room_id, len(joined_users), with_alias=False, allow_private=True, ) if not entry: -- cgit 1.5.1 From 70ea2f4e1df41458532c7964f4f707e04810e619 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 25 Feb 2019 19:15:36 +0000 Subject: switch from google.com to recaptcha.net for reCAPTCHA (#4731) * add trivial clarification about jemalloc * switch from google.com to recaptcha.net because https://developers.google.com/recaptcha/docs/faq#can-i-use-recaptcha-globally --- README.rst | 2 ++ synapse/config/captcha.py | 2 +- synapse/handlers/register.py | 4 ++-- synapse/rest/client/v2_alpha/auth.py | 2 +- synapse/static/client/register/index.html | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/README.rst b/README.rst index 9a7c04b55e..8e22109973 100644 --- a/README.rst +++ b/README.rst @@ -199,6 +199,8 @@ by installing the ``libjemalloc1`` package and adding this line to LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 +This can make a significant difference on Python 2.7 - it's unclear how +much of an improvement it provides on Python 3.x. Upgrading an existing Synapse ============================= diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 4064891ffb..d25196be08 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -47,5 +47,5 @@ class CaptchaConfig(Config): #captcha_bypass_secret: "YOUR_SECRET_HERE" # The API endpoint to use for verifying m.login.recaptcha responses. - recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" + recaptcha_siteverify_api: "https://www.recaptcha.net/recaptcha/api/siteverify" """ diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 24a4cb5a83..c0e06929bd 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -460,7 +460,7 @@ class RegistrationHandler(BaseHandler): lines = response.split('\n') json = { "valid": lines[0] == 'true', - "error_url": "http://www.google.com/recaptcha/api/challenge?" + + "error_url": "http://www.recaptcha.net/recaptcha/api/challenge?" + "error=%s" % lines[1] } defer.returnValue(json) @@ -471,7 +471,7 @@ class RegistrationHandler(BaseHandler): Used only by c/s api v1 """ data = yield self.captcha_client.post_urlencoded_get_raw( - "http://www.google.com:80/recaptcha/api/verify", + "http://www.recaptcha.net:80/recaptcha/api/verify", args={ 'privatekey': private_key, 'remoteip': ip_addr, diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index f7bb710642..ac035c7735 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -33,7 +33,7 @@ RECAPTCHA_TEMPLATE = """ Authentication - diff --git a/synapse/static/client/register/index.html b/synapse/static/client/register/index.html index 886f2edd1f..6edc4deb03 100644 --- a/synapse/static/client/register/index.html +++ b/synapse/static/client/register/index.html @@ -4,7 +4,7 @@ - + -- cgit 1.5.1 From 641c409e4e979c962d049b2b01404e4f3f2f75aa Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 25 Feb 2019 19:16:33 +0000 Subject: Fix ACME config for python 2. (#4717) Fixes #4675. --- changelog.d/4717.bugfix | 1 + synapse/config/tls.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4717.bugfix (limited to 'synapse') diff --git a/changelog.d/4717.bugfix b/changelog.d/4717.bugfix new file mode 100644 index 0000000000..79ab231477 --- /dev/null +++ b/changelog.d/4717.bugfix @@ -0,0 +1 @@ +Fix ACME config for python 2. diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 8d5d287357..40045de7ac 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -19,6 +19,8 @@ import warnings from datetime import datetime from hashlib import sha256 +import six + from unpaddedbase64 import encode_base64 from OpenSSL import crypto @@ -36,9 +38,11 @@ class TlsConfig(Config): acme_config = {} self.acme_enabled = acme_config.get("enabled", False) - self.acme_url = acme_config.get( + + # hyperlink complains on py2 if this is not a Unicode + self.acme_url = six.text_type(acme_config.get( "url", u"https://acme-v01.api.letsencrypt.org/directory" - ) + )) self.acme_port = acme_config.get("port", 80) self.acme_bind_addresses = acme_config.get("bind_addresses", ['::', '0.0.0.0']) self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30) @@ -55,7 +59,7 @@ class TlsConfig(Config): ) if not self.tls_private_key_file: raise ConfigError( - "tls_certificate_path must be specified if TLS-enabled listeners are " + "tls_private_key_path must be specified if TLS-enabled listeners are " "configured." ) -- cgit 1.5.1 From 899a119c2bc2ad9885efcf3f3958aec1cdccb464 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 19:17:22 +0000 Subject: Don't log stack trace when client has gone away during media download (#4738) * Don't log stack trace when client has gone away during media download * Newsfile * Fixup newsfile --- changelog.d/4738.misc | 1 + synapse/rest/media/v1/_base.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4738.misc (limited to 'synapse') diff --git a/changelog.d/4738.misc b/changelog.d/4738.misc new file mode 100644 index 0000000000..d5d0e27731 --- /dev/null +++ b/changelog.d/4738.misc @@ -0,0 +1 @@ +Cleanup request exception logging. diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py index efe42a429d..d16a30acd8 100644 --- a/synapse/rest/media/v1/_base.py +++ b/synapse/rest/media/v1/_base.py @@ -133,8 +133,15 @@ def respond_with_responder(request, responder, media_type, file_size, upload_nam logger.debug("Responding to media request with responder %s") add_file_headers(request, media_type, file_size, upload_name) - with responder: - yield responder.write_to_consumer(request) + try: + with responder: + yield responder.write_to_consumer(request) + except Exception as e: + # The majority of the time this will be due to the client having gone + # away. Unfortunately, Twisted simply throws a generic exception at us + # in that case. + logger.warning("Failed to write to consumer: %s %s", type(e), e) + finish_request(request) -- cgit 1.5.1 From 71669a0fba524e4d8cdb9e0076dca5d2770ef788 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 11:25:00 +0000 Subject: Address rich comments --- synapse/handlers/room_list.py | 36 ++++++++++++++++++++++++++++++------ synapse/server.pyi | 2 +- 2 files changed, 31 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 94cabc2a2e..a9725ade37 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -97,7 +97,23 @@ class RoomListHandler(BaseHandler): def _get_public_room_list(self, limit=None, since_token=None, search_filter=None, network_tuple=EMPTY_THIRD_PARTY_ID, - from_federation=False, timeout=None,): + from_federation=False, + timeout=None,): + """Generate a public room list. + + Args: + limit (int): Maximum amount of rooms to return. + since_token (str) + search_filter (dict): Dictionary to filter rooms by. + network_tuple (ThirdPartyInstanceID): Which public list to use. + This can be (None, None) to indicate the main list, or a particular + appservice and network id to use an appservice specific one. + Setting to None returns all public rooms across all lists. + from_federation (bool): Whether this request originated from a + federating server or a client. Used for room filtering. + timeout (int): Amount of seconds to wait for a response before + timing out. + """ if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: @@ -295,19 +311,28 @@ class RoomListHandler(BaseHandler): search_filter, from_federation=False): """Generate the entry for a room in the public room list and append it to the `chunk` if it matches the search filter + + Args: + room_id (str): The ID of the room. + num_joined_users (int): The number of joined users in the room. + chunk (list) + limit (int): Maximum amount of rooms to display. Function will + return if length of chunk is greater than limit + 1. + search_filter (dict) + from_federation (bool): Whether this request originated from a + federating server or a client. Used for room filtering. """ if limit and len(chunk) > limit + 1: # We've already got enough, so lets just drop it. return - result = yield self.generate_room_entry(room_id, - num_joined_users) + result = yield self.generate_room_entry(room_id, num_joined_users) if from_federation and not self.config.allow_non_federated_in_public_rooms: - if result["m.federate"] == False: + if result["m.federate"] is False: # This is a non-federating room and the config has chosen not # to show these rooms to other servers - chunk.append(None) + return elif result and _matches_room_entry(result, search_filter): chunk.append(result) @@ -318,7 +343,6 @@ class RoomListHandler(BaseHandler): Args: room_id (str): The room's ID. - disabled should be shown. num_joined_users (int): Number of users in the room. cache_context: Information for cached responses. with_alias (bool): Whether to return the room's aliases in the result. diff --git a/synapse/server.pyi b/synapse/server.pyi index 06cd083a74..fb8df56cd5 100644 --- a/synapse/server.pyi +++ b/synapse/server.pyi @@ -7,9 +7,9 @@ import synapse.handlers.auth import synapse.handlers.deactivate_account import synapse.handlers.device import synapse.handlers.e2e_keys +import synapse.handlers.message import synapse.handlers.room import synapse.handlers.room_member -import synapse.handlers.message import synapse.handlers.set_password import synapse.rest.media.v1.media_repository import synapse.server_notices.server_notices_manager -- cgit 1.5.1 From 6946c20111cb7c460deb340b61236dce9f4fd878 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 11:27:19 +0000 Subject: Result may be None --- synapse/handlers/room_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index a9725ade37..7c21f142a0 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -329,7 +329,7 @@ class RoomListHandler(BaseHandler): result = yield self.generate_room_entry(room_id, num_joined_users) if from_federation and not self.config.allow_non_federated_in_public_rooms: - if result["m.federate"] is False: + if not result or result["m.federate"] is False: # This is a non-federating room and the config has chosen not # to show these rooms to other servers return -- cgit 1.5.1 From 6728bf39405c52e4f2473c1e19a7648134361c15 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 11:52:52 +0000 Subject: Make not showing non-federated rooms the default --- synapse/config/room_directory.py | 12 ------------ synapse/handlers/room_list.py | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) (limited to 'synapse') diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py index 3322cf2eea..9b897abe3c 100644 --- a/synapse/config/room_directory.py +++ b/synapse/config/room_directory.py @@ -52,10 +52,6 @@ class RoomDirectoryConfig(Config): ) ] - self.allow_non_federated_in_public_rooms = config.get( - "allow_non_federated_in_public_rooms", True, - ) - def default_config(self, config_dir_path, server_name, **kwargs): return """ # The `alias_creation` option controls who's allowed to create aliases @@ -114,14 +110,6 @@ class RoomDirectoryConfig(Config): # alias: "*" # room_id: "*" # action: allow - - # Specify whether rooms that only allow local users to join should be - # shown in the federation public room directory. - # - # Note that this does not affect the room directory shown to users on - # this homeserver, only those on other homeservers. - # - #allow_non_federated_in_public_rooms: True """ def is_alias_creation_allowed(self, user_id, room_id, alias): diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 7c21f142a0..f1a51a7ca9 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -328,7 +328,7 @@ class RoomListHandler(BaseHandler): result = yield self.generate_room_entry(room_id, num_joined_users) - if from_federation and not self.config.allow_non_federated_in_public_rooms: + if from_federation: if not result or result["m.federate"] is False: # This is a non-federating room and the config has chosen not # to show these rooms to other servers -- cgit 1.5.1 From 40c2271680501c5ca2c29349d62df42d80b953db Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 12:04:34 +0000 Subject: Clean up room chunk logic --- synapse/handlers/room_list.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index f1a51a7ca9..e6038a1069 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -327,13 +327,15 @@ class RoomListHandler(BaseHandler): return result = yield self.generate_room_entry(room_id, num_joined_users) + if not result: + return - if from_federation: - if not result or result["m.federate"] is False: - # This is a non-federating room and the config has chosen not - # to show these rooms to other servers - return - elif result and _matches_room_entry(result, search_filter): + if from_federation and result["m.federate"] is False: + # This is a room that other servers cannot join. Do not show them + # this room. + return + + if _matches_room_entry(result, search_filter): chunk.append(result) @cachedInlineCallbacks(num_args=2, cache_context=True) -- cgit 1.5.1 From 16565e67dbc81ddf5fc16d39f82accbd508ac13f Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 12:12:48 +0000 Subject: Correct docstring types and chunk logic --- synapse/handlers/room_list.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index e6038a1069..7ab8476680 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -59,9 +59,9 @@ class RoomListHandler(BaseHandler): party network. A client can ask for a specific list or to return all. Args: - limit (int) - since_token (str) - search_filter (dict) + limit (int|None) + since_token (str|None) + search_filter (dict|None) network_tuple (ThirdPartyInstanceID): Which public list to use. This can be (None, None) to indicate the main list, or a particular appservice and network id to use an appservice specific one. @@ -102,16 +102,16 @@ class RoomListHandler(BaseHandler): """Generate a public room list. Args: - limit (int): Maximum amount of rooms to return. - since_token (str) - search_filter (dict): Dictionary to filter rooms by. + limit (int|None): Maximum amount of rooms to return. + since_token (str|None) + search_filter (dict|None): Dictionary to filter rooms by. network_tuple (ThirdPartyInstanceID): Which public list to use. This can be (None, None) to indicate the main list, or a particular appservice and network id to use an appservice specific one. Setting to None returns all public rooms across all lists. from_federation (bool): Whether this request originated from a federating server or a client. Used for room filtering. - timeout (int): Amount of seconds to wait for a response before + timeout (int|None): Amount of seconds to wait for a response before timing out. """ if since_token and since_token != "END": @@ -316,9 +316,9 @@ class RoomListHandler(BaseHandler): room_id (str): The ID of the room. num_joined_users (int): The number of joined users in the room. chunk (list) - limit (int): Maximum amount of rooms to display. Function will + limit (int|None): Maximum amount of rooms to display. Function will return if length of chunk is greater than limit + 1. - search_filter (dict) + search_filter (dict|None) from_federation (bool): Whether this request originated from a federating server or a client. Used for room filtering. """ @@ -330,7 +330,8 @@ class RoomListHandler(BaseHandler): if not result: return - if from_federation and result["m.federate"] is False: + if from_federation: + if "m.federate" in result and not result["m.federate"]: # This is a room that other servers cannot join. Do not show them # this room. return -- cgit 1.5.1 From a712aa3a9c5c58eca6ecfa0cb23636035fbbf449 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 12:13:55 +0000 Subject: Correct indent --- synapse/handlers/room_list.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 7ab8476680..0ce274e9ae 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -332,9 +332,9 @@ class RoomListHandler(BaseHandler): if from_federation: if "m.federate" in result and not result["m.federate"]: - # This is a room that other servers cannot join. Do not show them - # this room. - return + # This is a room that other servers cannot join. Do not show them + # this room. + return if _matches_room_entry(result, search_filter): chunk.append(result) -- cgit 1.5.1 From c4414768af5e2213b1ad2b16bc8e1b8062fd1e49 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 12:22:34 +0000 Subject: Cleaner chunk logic --- synapse/handlers/room_list.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 0ce274e9ae..c02fedcfe4 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -330,11 +330,10 @@ class RoomListHandler(BaseHandler): if not result: return - if from_federation: - if "m.federate" in result and not result["m.federate"]: - # This is a room that other servers cannot join. Do not show them - # this room. - return + if from_federation and not result.get("m.federate", True): + # This is a room that other servers cannot join. Do not show them + # this room. + return if _matches_room_entry(result, search_filter): chunk.append(result) -- cgit 1.5.1 From c74624a633ae883474a5ec3137bddd009165144c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 26 Feb 2019 13:20:38 +0000 Subject: Revert "Prevent showing non-fed rooms in fed /publicRooms" --- synapse/federation/transport/server.py | 3 +- synapse/groups/groups_server.py | 6 ++- synapse/handlers/room_list.py | 76 +++++----------------------------- synapse/server.pyi | 2 +- 4 files changed, 17 insertions(+), 70 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 5ba94be2ec..a2396ab466 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -736,8 +736,7 @@ class PublicRoomList(BaseFederationServlet): data = yield self.handler.get_local_public_room_list( limit, since_token, - network_tuple=network_tuple, - from_federation=True, + network_tuple=network_tuple ) defer.returnValue((200, data)) diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index a7eaead56b..633c865ed8 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -113,7 +113,8 @@ class GroupsServerHandler(object): room_id = room_entry["room_id"] joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), with_alias=False, allow_private=True, + room_id, len(joined_users), + with_alias=False, allow_private=True, ) entry = dict(entry) # so we don't change whats cached entry.pop("room_id", None) @@ -543,7 +544,8 @@ class GroupsServerHandler(object): joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), with_alias=False, allow_private=True, + room_id, len(joined_users), + with_alias=False, allow_private=True, ) if not entry: diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index c02fedcfe4..13e212d669 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -47,21 +47,19 @@ class RoomListHandler(BaseHandler): self.response_cache = ResponseCache(hs, "room_list") self.remote_response_cache = ResponseCache(hs, "remote_room_list", timeout_ms=30 * 1000) - self.config = hs.get_config() def get_local_public_room_list(self, limit=None, since_token=None, search_filter=None, - network_tuple=EMPTY_THIRD_PARTY_ID, - from_federation=False): + network_tuple=EMPTY_THIRD_PARTY_ID,): """Generate a local public room list. There are multiple different lists: the main one plus one per third party network. A client can ask for a specific list or to return all. Args: - limit (int|None) - since_token (str|None) - search_filter (dict|None) + limit (int) + since_token (str) + search_filter (dict) network_tuple (ThirdPartyInstanceID): Which public list to use. This can be (None, None) to indicate the main list, or a particular appservice and network id to use an appservice specific one. @@ -89,31 +87,14 @@ class RoomListHandler(BaseHandler): return self.response_cache.wrap( key, self._get_public_room_list, - limit, since_token, - network_tuple=network_tuple, from_federation=from_federation, + limit, since_token, network_tuple=network_tuple, ) @defer.inlineCallbacks def _get_public_room_list(self, limit=None, since_token=None, search_filter=None, network_tuple=EMPTY_THIRD_PARTY_ID, - from_federation=False, timeout=None,): - """Generate a public room list. - - Args: - limit (int|None): Maximum amount of rooms to return. - since_token (str|None) - search_filter (dict|None): Dictionary to filter rooms by. - network_tuple (ThirdPartyInstanceID): Which public list to use. - This can be (None, None) to indicate the main list, or a particular - appservice and network id to use an appservice specific one. - Setting to None returns all public rooms across all lists. - from_federation (bool): Whether this request originated from a - federating server or a client. Used for room filtering. - timeout (int|None): Amount of seconds to wait for a response before - timing out. - """ if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: @@ -236,8 +217,7 @@ class RoomListHandler(BaseHandler): yield concurrently_execute( lambda r: self._append_room_entry_to_chunk( r, rooms_to_num_joined[r], - chunk, limit, search_filter, - from_federation=from_federation, + chunk, limit, search_filter ), batch, 5, ) @@ -308,51 +288,23 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def _append_room_entry_to_chunk(self, room_id, num_joined_users, chunk, limit, - search_filter, from_federation=False): + search_filter): """Generate the entry for a room in the public room list and append it to the `chunk` if it matches the search filter - - Args: - room_id (str): The ID of the room. - num_joined_users (int): The number of joined users in the room. - chunk (list) - limit (int|None): Maximum amount of rooms to display. Function will - return if length of chunk is greater than limit + 1. - search_filter (dict|None) - from_federation (bool): Whether this request originated from a - federating server or a client. Used for room filtering. """ if limit and len(chunk) > limit + 1: # We've already got enough, so lets just drop it. return result = yield self.generate_room_entry(room_id, num_joined_users) - if not result: - return - - if from_federation and not result.get("m.federate", True): - # This is a room that other servers cannot join. Do not show them - # this room. - return - if _matches_room_entry(result, search_filter): + if result and _matches_room_entry(result, search_filter): chunk.append(result) - @cachedInlineCallbacks(num_args=2, cache_context=True) - def generate_room_entry(self, room_id, num_joined_users, - cache_context, with_alias=True, allow_private=False): + @cachedInlineCallbacks(num_args=1, cache_context=True) + def generate_room_entry(self, room_id, num_joined_users, cache_context, + with_alias=True, allow_private=False): """Returns the entry for a room - - Args: - room_id (str): The room's ID. - num_joined_users (int): Number of users in the room. - cache_context: Information for cached responses. - with_alias (bool): Whether to return the room's aliases in the result. - allow_private (bool): Whether invite-only rooms should be shown. - - Returns: - Deferred[dict|None]: Returns a room entry as a dictionary, or None if this - room was determined not to be shown publicly. """ result = { "room_id": room_id, @@ -366,7 +318,6 @@ class RoomListHandler(BaseHandler): event_map = yield self.store.get_events([ event_id for key, event_id in iteritems(current_state_ids) if key[0] in ( - EventTypes.Create, EventTypes.JoinRules, EventTypes.Name, EventTypes.Topic, @@ -383,17 +334,12 @@ class RoomListHandler(BaseHandler): } # Double check that this is actually a public room. - join_rules_event = current_state.get((EventTypes.JoinRules, "")) if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) if not allow_private and join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) - # Return whether this room is open to federation users or not - create_event = current_state.get((EventTypes.Create, "")) - result["m.federate"] = create_event.content.get("m.federate", True) - if with_alias: aliases = yield self.store.get_aliases_for_room( room_id, on_invalidate=cache_context.invalidate diff --git a/synapse/server.pyi b/synapse/server.pyi index fb8df56cd5..06cd083a74 100644 --- a/synapse/server.pyi +++ b/synapse/server.pyi @@ -7,9 +7,9 @@ import synapse.handlers.auth import synapse.handlers.deactivate_account import synapse.handlers.device import synapse.handlers.e2e_keys -import synapse.handlers.message import synapse.handlers.room import synapse.handlers.room_member +import synapse.handlers.message import synapse.handlers.set_password import synapse.rest.media.v1.media_repository import synapse.server_notices.server_notices_manager -- cgit 1.5.1 From 7a4632af9c4a26604b9d30f69eb81cb4630a82e7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 22 Feb 2019 12:11:43 +0000 Subject: Prevent showing non-fed rooms in fed /publicRooms --- synapse/federation/transport/server.py | 3 +- synapse/groups/groups_server.py | 6 ++-- synapse/handlers/room_list.py | 62 +++++++++++++++++++++++++++------- 3 files changed, 54 insertions(+), 17 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 7288d49074..6d4a26f595 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -697,7 +697,8 @@ class PublicRoomList(BaseFederationServlet): data = yield self.handler.get_local_public_room_list( limit, since_token, - network_tuple=network_tuple + network_tuple=network_tuple, + from_federation=True, ) defer.returnValue((200, data)) diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index 633c865ed8..a7eaead56b 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -113,8 +113,7 @@ class GroupsServerHandler(object): room_id = room_entry["room_id"] joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), - with_alias=False, allow_private=True, + room_id, len(joined_users), with_alias=False, allow_private=True, ) entry = dict(entry) # so we don't change whats cached entry.pop("room_id", None) @@ -544,8 +543,7 @@ class GroupsServerHandler(object): joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), - with_alias=False, allow_private=True, + room_id, len(joined_users), with_alias=False, allow_private=True, ) if not entry: diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index dc88620885..b722e2175f 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -50,16 +50,17 @@ class RoomListHandler(BaseHandler): def get_local_public_room_list(self, limit=None, since_token=None, search_filter=None, - network_tuple=EMPTY_THIRD_PARTY_ID,): + network_tuple=EMPTY_THIRD_PARTY_ID, + from_federation=False): """Generate a local public room list. There are multiple different lists: the main one plus one per third party network. A client can ask for a specific list or to return all. Args: - limit (int) - since_token (str) - search_filter (dict) + limit (int|None) + since_token (str|None) + search_filter (dict|None) network_tuple (ThirdPartyInstanceID): Which public list to use. This can be (None, None) to indicate the main list, or a particular appservice and network id to use an appservice specific one. @@ -81,13 +82,15 @@ class RoomListHandler(BaseHandler): return self.response_cache.wrap( key, self._get_public_room_list, - limit, since_token, network_tuple=network_tuple, + limit, since_token, + network_tuple=network_tuple, from_federation=from_federation, ) @defer.inlineCallbacks def _get_public_room_list(self, limit=None, since_token=None, search_filter=None, - network_tuple=EMPTY_THIRD_PARTY_ID,): + network_tuple=EMPTY_THIRD_PARTY_ID, + from_federation=False,): if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: @@ -207,7 +210,8 @@ class RoomListHandler(BaseHandler): yield concurrently_execute( lambda r: self._append_room_entry_to_chunk( r, rooms_to_num_joined[r], - chunk, limit, search_filter + chunk, limit, search_filter, + from_federation=from_federation, ), batch, 5, ) @@ -278,23 +282,51 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def _append_room_entry_to_chunk(self, room_id, num_joined_users, chunk, limit, - search_filter): + search_filter, from_federation=False): """Generate the entry for a room in the public room list and append it to the `chunk` if it matches the search filter + + Args: + room_id (str): The ID of the room. + num_joined_users (int): The number of joined users in the room. + chunk (list) + limit (int|None): Maximum amount of rooms to display. Function will + return if length of chunk is greater than limit + 1. + search_filter (dict|None) + from_federation (bool): Whether this request originated from a + federating server or a client. Used for room filtering. """ if limit and len(chunk) > limit + 1: # We've already got enough, so lets just drop it. return result = yield self.generate_room_entry(room_id, num_joined_users) + if not result: + return + + if from_federation and not result.get("m.federate", True): + # This is a room that other servers cannot join. Do not show them + # this room. + return - if result and _matches_room_entry(result, search_filter): + if _matches_room_entry(result, search_filter): chunk.append(result) - @cachedInlineCallbacks(num_args=1, cache_context=True) - def generate_room_entry(self, room_id, num_joined_users, cache_context, - with_alias=True, allow_private=False): + @cachedInlineCallbacks(num_args=2, cache_context=True) + def generate_room_entry(self, room_id, num_joined_users, + cache_context, with_alias=True, allow_private=False): """Returns the entry for a room + + Args: + room_id (str): The room's ID. + num_joined_users (int): Number of users in the room. + cache_context: Information for cached responses. + with_alias (bool): Whether to return the room's aliases in the result. + allow_private (bool): Whether invite-only rooms should be shown. + + Returns: + Deferred[dict|None]: Returns a room entry as a dictionary, or None if this + room was determined not to be shown publicly. """ result = { "room_id": room_id, @@ -308,6 +340,7 @@ class RoomListHandler(BaseHandler): event_map = yield self.store.get_events([ event_id for key, event_id in iteritems(current_state_ids) if key[0] in ( + EventTypes.Create, EventTypes.JoinRules, EventTypes.Name, EventTypes.Topic, @@ -324,12 +357,17 @@ class RoomListHandler(BaseHandler): } # Double check that this is actually a public room. + join_rules_event = current_state.get((EventTypes.JoinRules, "")) if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) if not allow_private and join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) + # Return whether this room is open to federation users or not + create_event = current_state.get((EventTypes.Create, "")) + result["m.federate"] = create_event.content.get("m.federate", True) + if with_alias: aliases = yield self.store.get_aliases_for_room( room_id, on_invalidate=cache_context.invalidate -- cgit 1.5.1 From 6fcb25202f34348e3411460496296252a2491bce Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 14:13:38 +0000 Subject: Put function def back to the way it was --- synapse/handlers/room_list.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index b722e2175f..4f51a464e7 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -312,9 +312,9 @@ class RoomListHandler(BaseHandler): if _matches_room_entry(result, search_filter): chunk.append(result) - @cachedInlineCallbacks(num_args=2, cache_context=True) - def generate_room_entry(self, room_id, num_joined_users, - cache_context, with_alias=True, allow_private=False): + @cachedInlineCallbacks(num_args=1, cache_context=True) + def generate_room_entry(self, room_id, num_joined_users, cache_context, + with_alias=True, allow_private=False): """Returns the entry for a room Args: -- cgit 1.5.1 From 313987187ee04dce5e70db17c1ab9377f283be7e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Feb 2019 15:04:34 +0000 Subject: Fix tightloop over connecting to replication server If the client failed to process incoming commands during the initial set up of the replication connection it would immediately disconnect and reconnect, resulting in a tightloop. This can happen, for example, when subscribing to a stream that has a row that is too long in the backlog. The fix here is to not consider the connection successfully set up until the client has succesfully subscribed and caught up with the streams. This ensures that the retry logic timers aren't reset until then, meaning that if an error does happen during start up the client will continue backing off before retrying again. --- docs/tcp_replication.rst | 4 +++- synapse/replication/tcp/client.py | 38 ++++++++++++++++++++++++++++++++++--- synapse/replication/tcp/commands.py | 5 ++++- 3 files changed, 42 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/docs/tcp_replication.rst b/docs/tcp_replication.rst index 73436cea62..75e723484c 100644 --- a/docs/tcp_replication.rst +++ b/docs/tcp_replication.rst @@ -188,7 +188,9 @@ RDATA (S) A single update in a stream POSITION (S) - The position of the stream has been updated + The position of the stream has been updated. Sent to the client after all + missing updates for a stream have been sent to the client and they're now + up to date. ERROR (S, C) There was an error diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 586dddb40b..914cd24b55 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -54,7 +54,6 @@ class ReplicationClientFactory(ReconnectingClientFactory): def buildProtocol(self, addr): logger.info("Connected to replication: %r", addr) - self.resetDelay() return ClientReplicationStreamProtocol( self.client_name, self.server_name, self._clock, self.handler ) @@ -90,15 +89,23 @@ class ReplicationClientHandler(object): # Used for tests. self.awaiting_syncs = {} + # Set of stream names that have been subscribe to, but haven't yet + # caught up with. This is used to track when the client has been fully + # connected to the remote. + self.streams_connecting = None + + # The factory used to create connections. + self.factory = None + def start_replication(self, hs): """Helper method to start a replication connection to the remote server using TCP. """ client_name = hs.config.worker_name - factory = ReplicationClientFactory(hs, client_name, self) + self.factory = ReplicationClientFactory(hs, client_name, self) host = hs.config.worker_replication_host port = hs.config.worker_replication_port - hs.get_reactor().connectTCP(host, port, factory) + hs.get_reactor().connectTCP(host, port, self.factory) def on_rdata(self, stream_name, token, rows): """Called when we get new replication data. By default this just pokes @@ -115,6 +122,12 @@ class ReplicationClientHandler(object): Can be overriden in subclasses to handle more. """ + # When we get a `POSITION` command it means we've finished getting + # missing updates for the given stream, and are now up to date. + self.streams_connecting.discard(stream_name) + if not self.streams_connecting: + self.finished_connecting() + return self.store.process_replication_rows(stream_name, token, []) def on_sync(self, data): @@ -140,6 +153,10 @@ class ReplicationClientHandler(object): args["account_data"] = user_account_data elif room_account_data: args["account_data"] = room_account_data + + # Record which streams we're in the process of subscribing to + self.streams_connecting = set(args.keys()) + return args def get_currently_syncing_users(self): @@ -204,3 +221,18 @@ class ReplicationClientHandler(object): for cmd in self.pending_commands: connection.send_command(cmd) self.pending_commands = [] + + # This will happen if we don't actually subscribe to any streams + if not self.streams_connecting: + self.finished_connecting() + + def finished_connecting(self): + """Called when we have successfully subscribed and caught up to all + streams we're interested in. + """ + logger.info("Finished connecting to server") + + # We don't reset the delay any earlier as otherwise if there is a + # problem during start up we'll end up tight looping connecting to the + # server. + self.factory.resetDelay() diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py index 327556f6a1..2098c32a77 100644 --- a/synapse/replication/tcp/commands.py +++ b/synapse/replication/tcp/commands.py @@ -127,8 +127,11 @@ class RdataCommand(Command): class PositionCommand(Command): - """Sent by the client to tell the client the stream postition without + """Sent by the server to tell the client the stream postition without needing to send an RDATA. + + Sent to the client after all missing updates for a stream have been sent + to the client and they're now up to date. """ NAME = "POSITION" -- cgit 1.5.1 From 25814921f1900e92b50830d6616762b174e82773 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Feb 2019 15:12:29 +0000 Subject: Increase the max delay between retry attempts Otherwise if you have many workers they can easily take out master with their connection attempts --- synapse/replication/tcp/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 914cd24b55..51f90655d0 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -39,7 +39,7 @@ class ReplicationClientFactory(ReconnectingClientFactory): Accepts a handler that will be called when new data is available or data is required. """ - maxDelay = 5 # Try at least once every N seconds + maxDelay = 30 # Try at least once every N seconds def __init__(self, hs, client_name, handler): self.client_name = client_name -- cgit 1.5.1 From f191be822b265b50339e17b2d95125f43dc700b3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 27 Feb 2019 10:21:49 +0000 Subject: Add database version to phonehome stats. (#4753) --- changelog.d/4753.misc | 1 + synapse/app/homeserver.py | 3 +++ synapse/storage/_base.py | 8 ++++++++ synapse/storage/engines/postgres.py | 25 +++++++++++++++++++++++++ synapse/storage/engines/sqlite.py | 9 +++++++++ 5 files changed, 46 insertions(+) create mode 100644 changelog.d/4753.misc (limited to 'synapse') diff --git a/changelog.d/4753.misc b/changelog.d/4753.misc new file mode 100644 index 0000000000..98532cc971 --- /dev/null +++ b/changelog.d/4753.misc @@ -0,0 +1 @@ +Add database version to phonehome stats. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 05a97979ec..e8b6cc3114 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -555,6 +555,9 @@ def run(hs): stats["memory_rss"] += process.memory_info().rss stats["cpu_average"] += int(process.cpu_percent(interval=None)) + stats["database_engine"] = hs.get_datastore().database_engine_name + stats["database_server_version"] = hs.get_datastore().get_server_version() + logger.info("Reporting stats to matrix.org: %s" % (stats,)) try: yield hs.get_simple_http_client().put_json( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5a80eef211..190be34fb1 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -1596,6 +1596,14 @@ class SQLBaseStore(object): return cls.cursor_to_dict(txn) + @property + def database_engine_name(self): + return self.database_engine.module.__name__ + + def get_server_version(self): + """Returns a string describing the server version number""" + return self.database_engine.server_version + class _RollbackButIsFineException(Exception): """ This exception is used to rollback a transaction without implying diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 4004427c7b..dc3238501c 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -23,6 +23,7 @@ class PostgresEngine(object): self.module = database_module self.module.extensions.register_type(self.module.extensions.UNICODE) self.synchronous_commit = database_config.get("synchronous_commit", True) + self._version = None # unknown as yet def check_database(self, txn): txn.execute("SHOW SERVER_ENCODING") @@ -87,3 +88,27 @@ class PostgresEngine(object): """ txn.execute("SELECT nextval('state_group_id_seq')") return txn.fetchone()[0] + + @property + def server_version(self): + """Returns a string giving the server version. For example: '8.1.5' + + Returns: + string + """ + # note that this is a bit of a hack because it relies on on_new_connection + # having been called at least once. Still, that should be a safe bet here. + numver = self._version + assert numver is not None + + # https://www.postgresql.org/docs/current/libpq-status.html#LIBPQ-PQSERVERVERSION + if numver >= 100000: + return "%i.%i" % ( + numver / 10000, numver % 10000, + ) + else: + return "%i.%i.%i" % ( + numver / 10000, + (numver % 10000) / 100, + numver % 100, + ) diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 059ab81055..1bcd5b99a4 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -70,6 +70,15 @@ class Sqlite3Engine(object): self._current_state_group_id += 1 return self._current_state_group_id + @property + def server_version(self): + """Gets a string giving the server version. For example: '3.22.0' + + Returns: + string + """ + return "%i.%i.%i" % self.module.sqlite_version_info + # Following functions taken from: https://github.com/coleifer/peewee -- cgit 1.5.1 From 6870fc496ff3da5075fec74e40515c03c929915f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Feb 2019 10:22:52 +0000 Subject: Move connecting logic into ClientReplicationStreamProtocol --- synapse/replication/tcp/client.py | 18 ------------------ synapse/replication/tcp/protocol.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 18 deletions(-) (limited to 'synapse') diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 51f90655d0..e558f90e1a 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -89,11 +89,6 @@ class ReplicationClientHandler(object): # Used for tests. self.awaiting_syncs = {} - # Set of stream names that have been subscribe to, but haven't yet - # caught up with. This is used to track when the client has been fully - # connected to the remote. - self.streams_connecting = None - # The factory used to create connections. self.factory = None @@ -122,12 +117,6 @@ class ReplicationClientHandler(object): Can be overriden in subclasses to handle more. """ - # When we get a `POSITION` command it means we've finished getting - # missing updates for the given stream, and are now up to date. - self.streams_connecting.discard(stream_name) - if not self.streams_connecting: - self.finished_connecting() - return self.store.process_replication_rows(stream_name, token, []) def on_sync(self, data): @@ -154,9 +143,6 @@ class ReplicationClientHandler(object): elif room_account_data: args["account_data"] = room_account_data - # Record which streams we're in the process of subscribing to - self.streams_connecting = set(args.keys()) - return args def get_currently_syncing_users(self): @@ -222,10 +208,6 @@ class ReplicationClientHandler(object): connection.send_command(cmd) self.pending_commands = [] - # This will happen if we don't actually subscribe to any streams - if not self.streams_connecting: - self.finished_connecting() - def finished_connecting(self): """Called when we have successfully subscribed and caught up to all streams we're interested in. diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 0b3fe6cbf5..6123c995b9 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -511,6 +511,11 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): self.server_name = server_name self.handler = handler + # Set of stream names that have been subscribe to, but haven't yet + # caught up with. This is used to track when the client has been fully + # connected to the remote. + self.streams_connecting = set() + # Map of stream to batched updates. See RdataCommand for info on how # batching works. self.pending_batches = {} @@ -533,6 +538,10 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): # We've now finished connecting to so inform the client handler self.handler.update_connection(self) + # This will happen if we don't actually subscribe to any streams + if not self.streams_connecting: + self.handler.finished_connecting() + def on_SERVER(self, cmd): if cmd.data != self.server_name: logger.error("[%s] Connected to wrong remote: %r", self.id(), cmd.data) @@ -562,6 +571,12 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): return self.handler.on_rdata(stream_name, cmd.token, rows) def on_POSITION(self, cmd): + # When we get a `POSITION` command it means we've finished getting + # missing updates for the given stream, and are now up to date. + self.streams_connecting.discard(cmd.stream_name) + if not self.streams_connecting: + self.handler.finished_connecting() + return self.handler.on_position(cmd.stream_name, cmd.token) def on_SYNC(self, cmd): @@ -578,6 +593,8 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): self.id(), stream_name, token ) + self.streams_connecting.add(stream_name) + self.send_command(ReplicateCommand(stream_name, token)) def on_connection_closed(self): -- cgit 1.5.1 From 6bb1c028f190d8ba561a6deaa474c060efb7c502 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Feb 2019 10:28:37 +0000 Subject: Limit cache invalidation replication line length (#4748) --- changelog.d/4748.misc | 1 + synapse/replication/tcp/protocol.py | 17 ++++++++++++++++- synapse/storage/_base.py | 15 +++++++++++---- 3 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 changelog.d/4748.misc (limited to 'synapse') diff --git a/changelog.d/4748.misc b/changelog.d/4748.misc new file mode 100644 index 0000000000..4dc18378e7 --- /dev/null +++ b/changelog.d/4748.misc @@ -0,0 +1 @@ +Improve replication performance by reducing cache invalidation traffic. diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 0b3fe6cbf5..429471c345 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -268,7 +268,17 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): if "\n" in string: raise Exception("Unexpected newline in command: %r", string) - self.sendLine(string.encode("utf-8")) + encoded_string = string.encode("utf-8") + + if len(encoded_string) > self.MAX_LENGTH: + raise Exception( + "Failed to send command %s as too long (%d > %d)" % ( + cmd.NAME, + len(encoded_string), self.MAX_LENGTH, + ) + ) + + self.sendLine(encoded_string) self.last_sent_command = self.clock.time_msec() @@ -361,6 +371,11 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): def id(self): return "%s-%s" % (self.name, self.conn_id) + def lineLengthExceeded(self, line): + """Called when we receive a line that is above the maximum line length + """ + self.send_error("Line length exceeded") + class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol): VALID_INBOUND_COMMANDS = VALID_CLIENT_COMMANDS diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 190be34fb1..a0333d5309 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -30,6 +30,7 @@ from synapse.api.errors import StoreError from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.types import get_domain_from_id +from synapse.util import batch_iter from synapse.util.caches.descriptors import Cache from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.stringutils import exception_to_unicode @@ -1327,10 +1328,16 @@ class SQLBaseStore(object): """ txn.call_after(self._invalidate_state_caches, room_id, members_changed) - keys = itertools.chain([room_id], members_changed) - self._send_invalidation_to_replication( - txn, _CURRENT_STATE_CACHE_NAME, keys, - ) + # We need to be careful that the size of the `members_changed` list + # isn't so large that it causes problems sending over replication, so we + # send them in chunks. + # Max line length is 16K, and max user ID length is 255, so 50 should + # be safe. + for chunk in batch_iter(members_changed, 50): + keys = itertools.chain([room_id], chunk) + self._send_invalidation_to_replication( + txn, _CURRENT_STATE_CACHE_NAME, keys, + ) def _invalidate_state_caches(self, room_id, members_changed): """Invalidates caches that are based on the current state, but does -- cgit 1.5.1 From 44a4d65586b231d62efea6e473e4343c17fca059 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 27 Feb 2019 10:48:34 +0000 Subject: 0.99.2rc1 --- CHANGES.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ changelog.d/4263.bugfix | 1 - changelog.d/4450.bugfix | 2 -- changelog.d/4541.feature | 1 - changelog.d/4632.feature | 1 - changelog.d/4635.misc | 1 - changelog.d/4642.feature | 1 - changelog.d/4643.misc | 1 - changelog.d/4644.misc | 1 - changelog.d/4647.feature | 1 - changelog.d/4651.bugfix | 1 - changelog.d/4652.feature | 1 - changelog.d/4657.misc | 1 - changelog.d/4666.feature | 1 - changelog.d/4667.bugfix | 1 - changelog.d/4668.misc | 1 - changelog.d/4669.misc | 1 - changelog.d/4670.feature | 1 - changelog.d/4671.misc | 1 - changelog.d/4674.feature | 1 - changelog.d/4676.misc | 1 - changelog.d/4677.misc | 1 - changelog.d/4680.bugfix | 3 --- changelog.d/4681.misc | 1 - changelog.d/4682.feature | 1 - changelog.d/4688.misc | 1 - changelog.d/4689.misc | 1 - changelog.d/4690.bugfix | 1 - changelog.d/4691.misc | 1 - changelog.d/4694.feature | 1 - changelog.d/4695.feature | 1 - changelog.d/4698.misc | 1 - changelog.d/4706.misc | 1 - changelog.d/4707.misc | 1 - changelog.d/4709.misc | 1 - changelog.d/4715.misc | 1 - changelog.d/4716.misc | 1 - changelog.d/4717.bugfix | 1 - changelog.d/4718.bugfix | 1 - changelog.d/4721.feature | 1 - changelog.d/4722.misc | 1 - changelog.d/4723.misc | 1 - changelog.d/4737.misc | 1 - changelog.d/4738.misc | 1 - changelog.d/4746.feature | 1 - changelog.d/4748.misc | 1 - changelog.d/4750.misc | 1 - changelog.d/4753.misc | 1 - debian/changelog | 5 ++-- synapse/__init__.py | 2 +- 50 files changed, 63 insertions(+), 53 deletions(-) delete mode 100644 changelog.d/4263.bugfix delete mode 100644 changelog.d/4450.bugfix delete mode 100644 changelog.d/4541.feature delete mode 100644 changelog.d/4632.feature delete mode 100644 changelog.d/4635.misc delete mode 100644 changelog.d/4642.feature delete mode 100644 changelog.d/4643.misc delete mode 100644 changelog.d/4644.misc delete mode 100644 changelog.d/4647.feature delete mode 100644 changelog.d/4651.bugfix delete mode 100644 changelog.d/4652.feature delete mode 100644 changelog.d/4657.misc delete mode 100644 changelog.d/4666.feature delete mode 100644 changelog.d/4667.bugfix delete mode 100644 changelog.d/4668.misc delete mode 100644 changelog.d/4669.misc delete mode 100644 changelog.d/4670.feature delete mode 100644 changelog.d/4671.misc delete mode 100644 changelog.d/4674.feature delete mode 100644 changelog.d/4676.misc delete mode 100644 changelog.d/4677.misc delete mode 100644 changelog.d/4680.bugfix delete mode 100644 changelog.d/4681.misc delete mode 100644 changelog.d/4682.feature delete mode 100644 changelog.d/4688.misc delete mode 100644 changelog.d/4689.misc delete mode 100644 changelog.d/4690.bugfix delete mode 100644 changelog.d/4691.misc delete mode 100644 changelog.d/4694.feature delete mode 100644 changelog.d/4695.feature delete mode 100644 changelog.d/4698.misc delete mode 100644 changelog.d/4706.misc delete mode 100644 changelog.d/4707.misc delete mode 100644 changelog.d/4709.misc delete mode 100644 changelog.d/4715.misc delete mode 100644 changelog.d/4716.misc delete mode 100644 changelog.d/4717.bugfix delete mode 100644 changelog.d/4718.bugfix delete mode 100644 changelog.d/4721.feature delete mode 100644 changelog.d/4722.misc delete mode 100644 changelog.d/4723.misc delete mode 100644 changelog.d/4737.misc delete mode 100644 changelog.d/4738.misc delete mode 100644 changelog.d/4746.feature delete mode 100644 changelog.d/4748.misc delete mode 100644 changelog.d/4750.misc delete mode 100644 changelog.d/4753.misc (limited to 'synapse') diff --git a/CHANGES.md b/CHANGES.md index f1a9d58e4d..22684420a4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,62 @@ +Synapse 0.99.2rc1 (2019-02-27) +============================== + +Features +-------- + +- Added an HAProxy example in the reverse proxy documentation. Contributed by Benoît S. (“Benpro”). ([\#4541](https://github.com/matrix-org/synapse/issues/4541)) +- Add basic optional sentry integration. ([\#4632](https://github.com/matrix-org/synapse/issues/4632), [\#4694](https://github.com/matrix-org/synapse/issues/4694)) +- Transfer bans on room upgrade. ([\#4642](https://github.com/matrix-org/synapse/issues/4642)) +- Add configurable room list publishing rules. ([\#4647](https://github.com/matrix-org/synapse/issues/4647)) +- Support .well-known delegation when issuing certificates through ACME. ([\#4652](https://github.com/matrix-org/synapse/issues/4652)) +- Allow registration and login to be handled by a worker instance. ([\#4666](https://github.com/matrix-org/synapse/issues/4666), [\#4670](https://github.com/matrix-org/synapse/issues/4670), [\#4682](https://github.com/matrix-org/synapse/issues/4682)) +- Reduce the overhead of creating outbound federation connections over TLS by caching the TLS client options. ([\#4674](https://github.com/matrix-org/synapse/issues/4674)) +- Add prometheus metrics for number of outgoing EDUs, by type. ([\#4695](https://github.com/matrix-org/synapse/issues/4695)) +- Return correct error code when inviting a remote user to a room whose homeserver does not support the room version. ([\#4721](https://github.com/matrix-org/synapse/issues/4721)) +- Prevent showing rooms to other servers that were set to not federate. ([\#4746](https://github.com/matrix-org/synapse/issues/4746)) + + +Bugfixes +-------- + +- Fix possible exception when paginating. ([\#4263](https://github.com/matrix-org/synapse/issues/4263)) +- The dependency checker now correctly reports a version mismatch for optional + dependencies, instead of reporting the dependency missing. ([\#4450](https://github.com/matrix-org/synapse/issues/4450)) +- Set CORS headers on .well-known requests. ([\#4651](https://github.com/matrix-org/synapse/issues/4651)) +- Fix kicking guest users on guest access revocation in worker mode. ([\#4667](https://github.com/matrix-org/synapse/issues/4667)) +- Fix an issue in the database migration script where the + `e2e_room_keys.is_verified` column wasn't considered as + a boolean. ([\#4680](https://github.com/matrix-org/synapse/issues/4680)) +- Fix TaskStopped exceptions in logs when outbound requests time out. ([\#4690](https://github.com/matrix-org/synapse/issues/4690)) +- Fix ACME config for python 2. ([\#4717](https://github.com/matrix-org/synapse/issues/4717)) +- Fix paginating over federation persisting incorrect state. ([\#4718](https://github.com/matrix-org/synapse/issues/4718)) + + +Internal Changes +---------------- + +- Run `black` to reformat user directory code. ([\#4635](https://github.com/matrix-org/synapse/issues/4635)) +- Reduce number of exceptions we log. ([\#4643](https://github.com/matrix-org/synapse/issues/4643), [\#4668](https://github.com/matrix-org/synapse/issues/4668)) +- Introduce upsert batching functionality in the database layer. ([\#4644](https://github.com/matrix-org/synapse/issues/4644)) +- Fix various spelling mistakes. ([\#4657](https://github.com/matrix-org/synapse/issues/4657)) +- Cleanup request exception logging. ([\#4669](https://github.com/matrix-org/synapse/issues/4669), [\#4737](https://github.com/matrix-org/synapse/issues/4737), [\#4738](https://github.com/matrix-org/synapse/issues/4738)) +- Improve replication performance by reducing cache invalidation traffic. ([\#4671](https://github.com/matrix-org/synapse/issues/4671), [\#4715](https://github.com/matrix-org/synapse/issues/4715), [\#4748](https://github.com/matrix-org/synapse/issues/4748)) +- Test against Postgres 9.5 as well as 9.4. ([\#4676](https://github.com/matrix-org/synapse/issues/4676)) +- Run unit tests against python 3.7. ([\#4677](https://github.com/matrix-org/synapse/issues/4677)) +- Attempt to clarify installation instructions/config. ([\#4681](https://github.com/matrix-org/synapse/issues/4681)) +- Clean up gitignores. ([\#4688](https://github.com/matrix-org/synapse/issues/4688)) +- Minor tweaks to acme docs. ([\#4689](https://github.com/matrix-org/synapse/issues/4689)) +- Improve the logging in the pusher process. ([\#4691](https://github.com/matrix-org/synapse/issues/4691)) +- Better checks on newsfragments. ([\#4698](https://github.com/matrix-org/synapse/issues/4698), [\#4750](https://github.com/matrix-org/synapse/issues/4750)) +- Avoid some redundant work when processing read receipts. ([\#4706](https://github.com/matrix-org/synapse/issues/4706)) +- Run `push_receipts_to_remotes` as background job. ([\#4707](https://github.com/matrix-org/synapse/issues/4707)) +- Add prometheus metrics for number of badge update pushes. ([\#4709](https://github.com/matrix-org/synapse/issues/4709)) +- Reduce pusher logging on startup ([\#4716](https://github.com/matrix-org/synapse/issues/4716)) +- Don't log exceptions when failing to fetch remote server keys. ([\#4722](https://github.com/matrix-org/synapse/issues/4722)) +- Correctly proxy exception in frontend_proxy worker. ([\#4723](https://github.com/matrix-org/synapse/issues/4723)) +- Add database version to phonehome stats. ([\#4753](https://github.com/matrix-org/synapse/issues/4753)) + + Synapse 0.99.1.1 (2019-02-14) ============================= diff --git a/changelog.d/4263.bugfix b/changelog.d/4263.bugfix deleted file mode 100644 index 3dc1d7c732..0000000000 --- a/changelog.d/4263.bugfix +++ /dev/null @@ -1 +0,0 @@ -Prevent crash on pagination. diff --git a/changelog.d/4450.bugfix b/changelog.d/4450.bugfix deleted file mode 100644 index b194e94c15..0000000000 --- a/changelog.d/4450.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -The dependency checker now correctly reports a version mismatch for optional -dependencies, instead of reporting the dependency missing. diff --git a/changelog.d/4541.feature b/changelog.d/4541.feature deleted file mode 100644 index 1d0e7bdfdc..0000000000 --- a/changelog.d/4541.feature +++ /dev/null @@ -1 +0,0 @@ -Added an HAProxy example in the reverse proxy documentation. Contributed by Benoît S. (“Benpro”). diff --git a/changelog.d/4632.feature b/changelog.d/4632.feature deleted file mode 100644 index d053ab5a25..0000000000 --- a/changelog.d/4632.feature +++ /dev/null @@ -1 +0,0 @@ -Add basic optional sentry integration diff --git a/changelog.d/4635.misc b/changelog.d/4635.misc deleted file mode 100644 index 0f45957b84..0000000000 --- a/changelog.d/4635.misc +++ /dev/null @@ -1 +0,0 @@ -Run `black` to reformat user directory code. diff --git a/changelog.d/4642.feature b/changelog.d/4642.feature deleted file mode 100644 index bfbf95bcbb..0000000000 --- a/changelog.d/4642.feature +++ /dev/null @@ -1 +0,0 @@ -Transfer bans on room upgrade. \ No newline at end of file diff --git a/changelog.d/4643.misc b/changelog.d/4643.misc deleted file mode 100644 index 556cdd2240..0000000000 --- a/changelog.d/4643.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce number of exceptions we log diff --git a/changelog.d/4644.misc b/changelog.d/4644.misc deleted file mode 100644 index 84137c3412..0000000000 --- a/changelog.d/4644.misc +++ /dev/null @@ -1 +0,0 @@ -Introduce upsert batching functionality in the database layer. diff --git a/changelog.d/4647.feature b/changelog.d/4647.feature deleted file mode 100644 index 5a5b1dcebb..0000000000 --- a/changelog.d/4647.feature +++ /dev/null @@ -1 +0,0 @@ -Add configurable room list publishing rules diff --git a/changelog.d/4651.bugfix b/changelog.d/4651.bugfix deleted file mode 100644 index 15cb1e58c4..0000000000 --- a/changelog.d/4651.bugfix +++ /dev/null @@ -1 +0,0 @@ -Set CORS headers on .well-known requests diff --git a/changelog.d/4652.feature b/changelog.d/4652.feature deleted file mode 100644 index ebe6880b21..0000000000 --- a/changelog.d/4652.feature +++ /dev/null @@ -1 +0,0 @@ -Support .well-known delegation when issuing certificates through ACME. diff --git a/changelog.d/4657.misc b/changelog.d/4657.misc deleted file mode 100644 index 8872765819..0000000000 --- a/changelog.d/4657.misc +++ /dev/null @@ -1 +0,0 @@ -Fix various spelling mistakes. diff --git a/changelog.d/4666.feature b/changelog.d/4666.feature deleted file mode 100644 index b3a3915eb0..0000000000 --- a/changelog.d/4666.feature +++ /dev/null @@ -1 +0,0 @@ -Allow registration and login to be handled by a worker instance. diff --git a/changelog.d/4667.bugfix b/changelog.d/4667.bugfix deleted file mode 100644 index 33ad00c137..0000000000 --- a/changelog.d/4667.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix kicking guest users on guest access revocation in worker mode. diff --git a/changelog.d/4668.misc b/changelog.d/4668.misc deleted file mode 100644 index 556cdd2240..0000000000 --- a/changelog.d/4668.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce number of exceptions we log diff --git a/changelog.d/4669.misc b/changelog.d/4669.misc deleted file mode 100644 index d5d0e27731..0000000000 --- a/changelog.d/4669.misc +++ /dev/null @@ -1 +0,0 @@ -Cleanup request exception logging. diff --git a/changelog.d/4670.feature b/changelog.d/4670.feature deleted file mode 100644 index b3a3915eb0..0000000000 --- a/changelog.d/4670.feature +++ /dev/null @@ -1 +0,0 @@ -Allow registration and login to be handled by a worker instance. diff --git a/changelog.d/4671.misc b/changelog.d/4671.misc deleted file mode 100644 index 4dc18378e7..0000000000 --- a/changelog.d/4671.misc +++ /dev/null @@ -1 +0,0 @@ -Improve replication performance by reducing cache invalidation traffic. diff --git a/changelog.d/4674.feature b/changelog.d/4674.feature deleted file mode 100644 index 84630bb201..0000000000 --- a/changelog.d/4674.feature +++ /dev/null @@ -1 +0,0 @@ -Reduce the overhead of creating outbound federation connections over TLS by caching the TLS client options. diff --git a/changelog.d/4676.misc b/changelog.d/4676.misc deleted file mode 100644 index a250558e69..0000000000 --- a/changelog.d/4676.misc +++ /dev/null @@ -1 +0,0 @@ -Test against Postgres 9.5 as well as 9.4 diff --git a/changelog.d/4677.misc b/changelog.d/4677.misc deleted file mode 100644 index 6f4596be4a..0000000000 --- a/changelog.d/4677.misc +++ /dev/null @@ -1 +0,0 @@ -Run unit tests against python 3.7. diff --git a/changelog.d/4680.bugfix b/changelog.d/4680.bugfix deleted file mode 100644 index 4aad8ecde3..0000000000 --- a/changelog.d/4680.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue in the database migration script where the -`e2e_room_keys.is_verified` column wasn't considered as -a boolean diff --git a/changelog.d/4681.misc b/changelog.d/4681.misc deleted file mode 100644 index 37d3588804..0000000000 --- a/changelog.d/4681.misc +++ /dev/null @@ -1 +0,0 @@ -Attempt to clarify installation instructions/config diff --git a/changelog.d/4682.feature b/changelog.d/4682.feature deleted file mode 100644 index b3a3915eb0..0000000000 --- a/changelog.d/4682.feature +++ /dev/null @@ -1 +0,0 @@ -Allow registration and login to be handled by a worker instance. diff --git a/changelog.d/4688.misc b/changelog.d/4688.misc deleted file mode 100644 index 24cd2eb424..0000000000 --- a/changelog.d/4688.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up gitignores diff --git a/changelog.d/4689.misc b/changelog.d/4689.misc deleted file mode 100644 index 15c4d9404b..0000000000 --- a/changelog.d/4689.misc +++ /dev/null @@ -1 +0,0 @@ -Minor tweaks to acme docs. diff --git a/changelog.d/4690.bugfix b/changelog.d/4690.bugfix deleted file mode 100644 index e4cfc5e413..0000000000 --- a/changelog.d/4690.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix TaskStopped exceptions in logs when outbound requests time out. \ No newline at end of file diff --git a/changelog.d/4691.misc b/changelog.d/4691.misc deleted file mode 100644 index 8eb825edf0..0000000000 --- a/changelog.d/4691.misc +++ /dev/null @@ -1 +0,0 @@ -Improve the logging in the pusher process. diff --git a/changelog.d/4694.feature b/changelog.d/4694.feature deleted file mode 100644 index d053ab5a25..0000000000 --- a/changelog.d/4694.feature +++ /dev/null @@ -1 +0,0 @@ -Add basic optional sentry integration diff --git a/changelog.d/4695.feature b/changelog.d/4695.feature deleted file mode 100644 index 3816c9dec8..0000000000 --- a/changelog.d/4695.feature +++ /dev/null @@ -1 +0,0 @@ -Add prometheus metrics for number of outgoing EDUs, by type. diff --git a/changelog.d/4698.misc b/changelog.d/4698.misc deleted file mode 100644 index 9dea5dd2be..0000000000 --- a/changelog.d/4698.misc +++ /dev/null @@ -1 +0,0 @@ -Better checks on newsfragments. diff --git a/changelog.d/4706.misc b/changelog.d/4706.misc deleted file mode 100644 index 73d1ddcc56..0000000000 --- a/changelog.d/4706.misc +++ /dev/null @@ -1 +0,0 @@ -Avoid some redundant work when processing read receipts diff --git a/changelog.d/4707.misc b/changelog.d/4707.misc deleted file mode 100644 index ef0772b9af..0000000000 --- a/changelog.d/4707.misc +++ /dev/null @@ -1 +0,0 @@ -Run push_receipts_to_remotes as background job. diff --git a/changelog.d/4709.misc b/changelog.d/4709.misc deleted file mode 100644 index ca47a6f327..0000000000 --- a/changelog.d/4709.misc +++ /dev/null @@ -1 +0,0 @@ -Add prometheus metrics for number of badge update pushes. diff --git a/changelog.d/4715.misc b/changelog.d/4715.misc deleted file mode 100644 index 4dc18378e7..0000000000 --- a/changelog.d/4715.misc +++ /dev/null @@ -1 +0,0 @@ -Improve replication performance by reducing cache invalidation traffic. diff --git a/changelog.d/4716.misc b/changelog.d/4716.misc deleted file mode 100644 index 5935f3af24..0000000000 --- a/changelog.d/4716.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce pusher logging on startup diff --git a/changelog.d/4717.bugfix b/changelog.d/4717.bugfix deleted file mode 100644 index 79ab231477..0000000000 --- a/changelog.d/4717.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix ACME config for python 2. diff --git a/changelog.d/4718.bugfix b/changelog.d/4718.bugfix deleted file mode 100644 index a7d1963ee1..0000000000 --- a/changelog.d/4718.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix paginating over federation persisting incorrect state. diff --git a/changelog.d/4721.feature b/changelog.d/4721.feature deleted file mode 100644 index f932843ce7..0000000000 --- a/changelog.d/4721.feature +++ /dev/null @@ -1 +0,0 @@ -Return correct error code when inviting a remote user to a room whose homeserver does not support the room version. diff --git a/changelog.d/4722.misc b/changelog.d/4722.misc deleted file mode 100644 index e9158c4dc2..0000000000 --- a/changelog.d/4722.misc +++ /dev/null @@ -1 +0,0 @@ -Don't log exceptions when failing to fetch remote server keys diff --git a/changelog.d/4723.misc b/changelog.d/4723.misc deleted file mode 100644 index 96958036ca..0000000000 --- a/changelog.d/4723.misc +++ /dev/null @@ -1 +0,0 @@ -Correctly proxy exception in frontend_proxy worker diff --git a/changelog.d/4737.misc b/changelog.d/4737.misc deleted file mode 100644 index d5d0e27731..0000000000 --- a/changelog.d/4737.misc +++ /dev/null @@ -1 +0,0 @@ -Cleanup request exception logging. diff --git a/changelog.d/4738.misc b/changelog.d/4738.misc deleted file mode 100644 index d5d0e27731..0000000000 --- a/changelog.d/4738.misc +++ /dev/null @@ -1 +0,0 @@ -Cleanup request exception logging. diff --git a/changelog.d/4746.feature b/changelog.d/4746.feature deleted file mode 100644 index 97c253eccf..0000000000 --- a/changelog.d/4746.feature +++ /dev/null @@ -1 +0,0 @@ -Prevent showing rooms to other servers that were set to not federate. \ No newline at end of file diff --git a/changelog.d/4748.misc b/changelog.d/4748.misc deleted file mode 100644 index 4dc18378e7..0000000000 --- a/changelog.d/4748.misc +++ /dev/null @@ -1 +0,0 @@ -Improve replication performance by reducing cache invalidation traffic. diff --git a/changelog.d/4750.misc b/changelog.d/4750.misc deleted file mode 100644 index 3bb9c48f1a..0000000000 --- a/changelog.d/4750.misc +++ /dev/null @@ -1 +0,0 @@ -Better checks on newsfragments. \ No newline at end of file diff --git a/changelog.d/4753.misc b/changelog.d/4753.misc deleted file mode 100644 index 98532cc971..0000000000 --- a/changelog.d/4753.misc +++ /dev/null @@ -1 +0,0 @@ -Add database version to phonehome stats. diff --git a/debian/changelog b/debian/changelog index 7631406a68..8a59c708d7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,9 @@ -matrix-synapse-py3 (0.99.2) UNRELEASED; urgency=medium +matrix-synapse-py3 (0.99.2rc1) stable; urgency=medium * Fix overwriting of config settings on upgrade. + * New synapse release 0.99.2rc1. - -- Synapse Packaging team Wed, 20 Feb 2019 17:11:25 +0000 + -- Synapse Packaging team Wed, 27 Feb 2019 10:45:58 +0000 matrix-synapse-py3 (0.99.1.1) stable; urgency=medium diff --git a/synapse/__init__.py b/synapse/__init__.py index 2004375f98..29b1fe4c03 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.1.1" +__version__ = "0.99.2rc1" -- cgit 1.5.1 From 4cff9376f7b07d3c60b1f882cde5848149862dd9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Feb 2019 13:43:53 +0000 Subject: Move server key queries to federation reader --- docs/workers.rst | 1 + synapse/app/federation_reader.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/docs/workers.rst b/docs/workers.rst index 3ba5879f76..7552455a4e 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -182,6 +182,7 @@ endpoints matching the following regular expressions:: ^/_matrix/federation/v1/event_auth/ ^/_matrix/federation/v1/exchange_third_party_invite/ ^/_matrix/federation/v1/send/ + ^/_matrix/key/v2/query The above endpoints should all be routed to the federation_reader worker by the reverse-proxy configuration. diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index b116c17669..7da79dc827 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -21,7 +21,7 @@ from twisted.web.resource import NoResource import synapse from synapse import events -from synapse.api.urls import FEDERATION_PREFIX +from synapse.api.urls import FEDERATION_PREFIX, SERVER_KEY_V2_PREFIX from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig @@ -44,6 +44,7 @@ from synapse.replication.slave.storage.registration import SlavedRegistrationSto from synapse.replication.slave.storage.room import RoomStore from synapse.replication.slave.storage.transactions import SlavedTransactionStore from synapse.replication.tcp.client import ReplicationClientHandler +from synapse.rest.key.v2 import KeyApiV2Resource from synapse.server import HomeServer from synapse.storage.engines import create_engine from synapse.util.httpresourcetree import create_resource_tree @@ -99,6 +100,9 @@ class FederationReaderServer(HomeServer): ), }) + if name in ["keys", "federation"]: + resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( -- cgit 1.5.1 From 1e315017d3c0dcde20e781fb3c87bc0e54c53cd1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Feb 2019 13:53:46 +0000 Subject: When presence is enabled don't send over replication --- synapse/federation/federation_server.py | 3 +++ synapse/replication/slave/storage/presence.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 569eb277a9..81f3b4b1ff 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -886,6 +886,9 @@ class ReplicationFederationHandlerRegistry(FederationHandlerRegistry): def on_edu(self, edu_type, origin, content): """Overrides FederationHandlerRegistry """ + if not self.config.use_presence and edu_type == "m.presence": + return + handler = self.edu_handlers.get(edu_type) if handler: return super(ReplicationFederationHandlerRegistry, self).on_edu( diff --git a/synapse/replication/slave/storage/presence.py b/synapse/replication/slave/storage/presence.py index 92447b00d4..9e530defe0 100644 --- a/synapse/replication/slave/storage/presence.py +++ b/synapse/replication/slave/storage/presence.py @@ -54,8 +54,11 @@ class SlavedPresenceStore(BaseSlavedStore): def stream_positions(self): result = super(SlavedPresenceStore, self).stream_positions() - position = self._presence_id_gen.get_current_token() - result["presence"] = position + + if self.hs.config.use_presence: + position = self._presence_id_gen.get_current_token() + result["presence"] = position + return result def process_replication_rows(self, stream_name, token, rows): -- cgit 1.5.1 From 54f9ce11a7100e7207bb068a84dfaba886995c2c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Feb 2019 14:26:08 +0000 Subject: Move /account/3pid to client_reader --- docs/workers.rst | 1 + synapse/app/client_reader.py | 2 ++ synapse/storage/registration.py | 66 ++++++++++++++++++++--------------------- 3 files changed, 36 insertions(+), 33 deletions(-) (limited to 'synapse') diff --git a/docs/workers.rst b/docs/workers.rst index 3ba5879f76..0170fba898 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -223,6 +223,7 @@ following regular expressions:: ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$ ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$ ^/_matrix/client/(api/v1|r0|unstable)/login$ + ^/_matrix/client/(api/v1|r0|unstable)/account/3pid$ Additionally, the following REST endpoints can be handled, but all requests must be routed to the same instance:: diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 043b48f8f3..5070094cad 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -48,6 +48,7 @@ from synapse.rest.client.v1.room import ( RoomMemberListRestServlet, RoomStateRestServlet, ) +from synapse.rest.client.v2_alpha.account import ThreepidRestServlet from synapse.rest.client.v2_alpha.register import RegisterRestServlet from synapse.server import HomeServer from synapse.storage.engines import create_engine @@ -96,6 +97,7 @@ class ClientReaderServer(HomeServer): RoomEventContextServlet(self).register(resource) RegisterRestServlet(self).register(resource) LoginRestServlet(self).register(resource) + ThreepidRestServlet(self).register(resource) resources.update({ "/_matrix/client/r0": resource, diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 9b9572890b..9b6c28892c 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -295,6 +295,39 @@ class RegistrationWorkerStore(SQLBaseStore): return ret['user_id'] return None + @defer.inlineCallbacks + def user_add_threepid(self, user_id, medium, address, validated_at, added_at): + yield self._simple_upsert("user_threepids", { + "medium": medium, + "address": address, + }, { + "user_id": user_id, + "validated_at": validated_at, + "added_at": added_at, + }) + + @defer.inlineCallbacks + def user_get_threepids(self, user_id): + ret = yield self._simple_select_list( + "user_threepids", { + "user_id": user_id + }, + ['medium', 'address', 'validated_at', 'added_at'], + 'user_get_threepids' + ) + defer.returnValue(ret) + + def user_delete_threepid(self, user_id, medium, address): + return self._simple_delete( + "user_threepids", + keyvalues={ + "user_id": user_id, + "medium": medium, + "address": address, + }, + desc="user_delete_threepids", + ) + class RegistrationStore(RegistrationWorkerStore, background_updates.BackgroundUpdateStore): @@ -632,39 +665,6 @@ class RegistrationStore(RegistrationWorkerStore, defer.returnValue(res if res else False) - @defer.inlineCallbacks - def user_add_threepid(self, user_id, medium, address, validated_at, added_at): - yield self._simple_upsert("user_threepids", { - "medium": medium, - "address": address, - }, { - "user_id": user_id, - "validated_at": validated_at, - "added_at": added_at, - }) - - @defer.inlineCallbacks - def user_get_threepids(self, user_id): - ret = yield self._simple_select_list( - "user_threepids", { - "user_id": user_id - }, - ['medium', 'address', 'validated_at', 'added_at'], - 'user_get_threepids' - ) - defer.returnValue(ret) - - def user_delete_threepid(self, user_id, medium, address): - return self._simple_delete( - "user_threepids", - keyvalues={ - "user_id": user_id, - "medium": medium, - "address": address, - }, - desc="user_delete_threepids", - ) - @defer.inlineCallbacks def save_or_get_3pid_guest_access_token( self, medium, address, access_token, inviter_user_id -- cgit 1.5.1 From f2a753ea3833a0f9ab8f1d6ca5e7d0282adb109d Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 27 Feb 2019 13:03:14 -0800 Subject: Move from TravisCI to BuildKite (#4752) --- .buildkite/.env | 13 +++ .buildkite/docker-compose.py27.pg94.yaml | 21 +++++ .buildkite/docker-compose.py27.pg95.yaml | 21 +++++ .buildkite/docker-compose.py35.pg94.yaml | 21 +++++ .buildkite/docker-compose.py35.pg95.yaml | 21 +++++ .buildkite/docker-compose.py37.pg11.yaml | 21 +++++ .buildkite/docker-compose.py37.pg95.yaml | 21 +++++ .buildkite/pipeline.yml | 149 +++++++++++++++++++++++++++++++ .travis.yml | 97 -------------------- MANIFEST.in | 1 + changelog.d/4752.misc | 1 + synapse/server.pyi | 2 +- tests/utils.py | 44 ++++++--- 13 files changed, 325 insertions(+), 108 deletions(-) create mode 100644 .buildkite/.env create mode 100644 .buildkite/docker-compose.py27.pg94.yaml create mode 100644 .buildkite/docker-compose.py27.pg95.yaml create mode 100644 .buildkite/docker-compose.py35.pg94.yaml create mode 100644 .buildkite/docker-compose.py35.pg95.yaml create mode 100644 .buildkite/docker-compose.py37.pg11.yaml create mode 100644 .buildkite/docker-compose.py37.pg95.yaml create mode 100644 .buildkite/pipeline.yml delete mode 100644 .travis.yml create mode 100644 changelog.d/4752.misc (limited to 'synapse') diff --git a/.buildkite/.env b/.buildkite/.env new file mode 100644 index 0000000000..85b102d07f --- /dev/null +++ b/.buildkite/.env @@ -0,0 +1,13 @@ +CI +BUILDKITE +BUILDKITE_BUILD_NUMBER +BUILDKITE_BRANCH +BUILDKITE_BUILD_NUMBER +BUILDKITE_JOB_ID +BUILDKITE_BUILD_URL +BUILDKITE_PROJECT_SLUG +BUILDKITE_COMMIT +BUILDKITE_PULL_REQUEST +BUILDKITE_TAG +CODECOV_TOKEN +TRIAL_FLAGS diff --git a/.buildkite/docker-compose.py27.pg94.yaml b/.buildkite/docker-compose.py27.pg94.yaml new file mode 100644 index 0000000000..2d4b9eadd9 --- /dev/null +++ b/.buildkite/docker-compose.py27.pg94.yaml @@ -0,0 +1,21 @@ +version: '3.1' + +services: + + postgres: + image: postgres:9.4 + environment: + POSTGRES_PASSWORD: postgres + + testenv: + image: python:2.7 + depends_on: + - postgres + env_file: .env + environment: + SYNAPSE_POSTGRES_HOST: postgres + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + working_dir: /app + volumes: + - ..:/app diff --git a/.buildkite/docker-compose.py27.pg95.yaml b/.buildkite/docker-compose.py27.pg95.yaml new file mode 100644 index 0000000000..c6a41f1da0 --- /dev/null +++ b/.buildkite/docker-compose.py27.pg95.yaml @@ -0,0 +1,21 @@ +version: '3.1' + +services: + + postgres: + image: postgres:9.5 + environment: + POSTGRES_PASSWORD: postgres + + testenv: + image: python:2.7 + depends_on: + - postgres + env_file: .env + environment: + SYNAPSE_POSTGRES_HOST: postgres + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + working_dir: /app + volumes: + - ..:/app diff --git a/.buildkite/docker-compose.py35.pg94.yaml b/.buildkite/docker-compose.py35.pg94.yaml new file mode 100644 index 0000000000..978aedd115 --- /dev/null +++ b/.buildkite/docker-compose.py35.pg94.yaml @@ -0,0 +1,21 @@ +version: '3.1' + +services: + + postgres: + image: postgres:9.4 + environment: + POSTGRES_PASSWORD: postgres + + testenv: + image: python:3.5 + depends_on: + - postgres + env_file: .env + environment: + SYNAPSE_POSTGRES_HOST: postgres + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + working_dir: /app + volumes: + - ..:/app diff --git a/.buildkite/docker-compose.py35.pg95.yaml b/.buildkite/docker-compose.py35.pg95.yaml new file mode 100644 index 0000000000..2f14387fbc --- /dev/null +++ b/.buildkite/docker-compose.py35.pg95.yaml @@ -0,0 +1,21 @@ +version: '3.1' + +services: + + postgres: + image: postgres:9.5 + environment: + POSTGRES_PASSWORD: postgres + + testenv: + image: python:3.5 + depends_on: + - postgres + env_file: .env + environment: + SYNAPSE_POSTGRES_HOST: postgres + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + working_dir: /app + volumes: + - ..:/app diff --git a/.buildkite/docker-compose.py37.pg11.yaml b/.buildkite/docker-compose.py37.pg11.yaml new file mode 100644 index 0000000000..f3eec05ceb --- /dev/null +++ b/.buildkite/docker-compose.py37.pg11.yaml @@ -0,0 +1,21 @@ +version: '3.1' + +services: + + postgres: + image: postgres:11 + environment: + POSTGRES_PASSWORD: postgres + + testenv: + image: python:3.7 + depends_on: + - postgres + env_file: .env + environment: + SYNAPSE_POSTGRES_HOST: postgres + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + working_dir: /app + volumes: + - ..:/app diff --git a/.buildkite/docker-compose.py37.pg95.yaml b/.buildkite/docker-compose.py37.pg95.yaml new file mode 100644 index 0000000000..2a41db8eba --- /dev/null +++ b/.buildkite/docker-compose.py37.pg95.yaml @@ -0,0 +1,21 @@ +version: '3.1' + +services: + + postgres: + image: postgres:9.5 + environment: + POSTGRES_PASSWORD: postgres + + testenv: + image: python:3.7 + depends_on: + - postgres + env_file: .env + environment: + SYNAPSE_POSTGRES_HOST: postgres + SYNAPSE_POSTGRES_USER: postgres + SYNAPSE_POSTGRES_PASSWORD: postgres + working_dir: /app + volumes: + - ..:/app diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000000..24f22c85b4 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,149 @@ +env: + CODECOV_TOKEN: "2dd7eb9b-0eda-45fe-a47c-9b5ac040045f" + +steps: + - command: + - "python -m pip install tox" + - "tox -e pep8" + label: "\U0001F9F9 PEP-8" + plugins: + - docker#v3.0.1: + image: "python:3.6" + + - command: + - "python -m pip install tox" + - "tox -e packaging" + label: "\U0001F9F9 packaging" + plugins: + - docker#v3.0.1: + image: "python:3.6" + + - command: + - "python -m pip install tox" + - "tox -e check_isort" + label: "\U0001F9F9 isort" + plugins: + - docker#v3.0.1: + image: "python:3.6" + + - command: + - "python -m pip install tox" + - "scripts-dev/check-newsfragment" + label: ":newspaper: Newsfile" + branches: "!master !develop !release-*" + plugins: + - docker#v3.0.1: + image: "python:3.6" + propagate-environment: true + + - wait + + - command: + - "python -m pip install tox" + - "tox -e py27,codecov" + label: ":python: 2.7 / SQLite" + env: + TRIAL_FLAGS: "-j 2" + plugins: + - docker#v3.0.1: + image: "python:2.7" + propagate-environment: true + + - command: + - "python -m pip install tox" + - "tox -e py35,codecov" + label: ":python: 3.5 / SQLite" + env: + TRIAL_FLAGS: "-j 2" + plugins: + - docker#v3.0.1: + image: "python:3.5" + propagate-environment: true + + - command: + - "python -m pip install tox" + - "tox -e py36,codecov" + label: ":python: 3.6 / SQLite" + env: + TRIAL_FLAGS: "-j 2" + plugins: + - docker#v3.0.1: + image: "python:3.6" + propagate-environment: true + + - command: + - "python -m pip install tox" + - "tox -e py37,codecov" + label: ":python: 3.7 / SQLite" + env: + TRIAL_FLAGS: "-j 2" + plugins: + - docker#v3.0.1: + image: "python:3.7" + propagate-environment: true + + - label: ":python: 2.7 / :postgres: 9.4" + env: + TRIAL_FLAGS: "-j 4" + command: + - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'" + plugins: + - docker-compose#v2.1.0: + run: testenv + config: + - .buildkite/docker-compose.py27.pg94.yaml + + - label: ":python: 2.7 / :postgres: 9.5" + env: + TRIAL_FLAGS: "-j 4" + command: + - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'" + plugins: + - docker-compose#v2.1.0: + run: testenv + config: + - .buildkite/docker-compose.py27.pg95.yaml + + - label: ":python: 3.5 / :postgres: 9.4" + env: + TRIAL_FLAGS: "-j 4" + command: + - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'" + plugins: + - docker-compose#v2.1.0: + run: testenv + config: + - .buildkite/docker-compose.py35.pg94.yaml + + - label: ":python: 3.5 / :postgres: 9.5" + env: + TRIAL_FLAGS: "-j 4" + command: + - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'" + plugins: + - docker-compose#v2.1.0: + run: testenv + config: + - .buildkite/docker-compose.py35.pg95.yaml + + - label: ":python: 3.7 / :postgres: 9.5" + env: + TRIAL_FLAGS: "-j 4" + command: + - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'" + plugins: + - docker-compose#v2.1.0: + run: testenv + config: + - .buildkite/docker-compose.py37.pg95.yaml + + - label: ":python: 3.7 / :postgres: 11" + env: + TRIAL_FLAGS: "-j 4" + command: + - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'" + plugins: + - docker-compose#v2.1.0: + run: testenv + config: + - .buildkite/docker-compose.py37.pg11.yaml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0d0fa7082a..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,97 +0,0 @@ -dist: xenial -language: python - -cache: - directories: - # we only bother to cache the wheels; parts of the http cache get - # invalidated every build (because they get served with a max-age of 600 - # seconds), which means that we end up re-uploading the whole cache for - # every build, which is time-consuming In any case, it's not obvious that - # downloading the cache from S3 would be much faster than downloading the - # originals from pypi. - # - - $HOME/.cache/pip/wheels - -# don't clone the whole repo history, one commit will do -git: - depth: 1 - -# only build branches we care about (PRs are built seperately) -branches: - only: - - master - - develop - - /^release-v/ - - rav/pg95 - -# When running the tox environments that call Twisted Trial, we can pass the -j -# flag to run the tests concurrently. We set this to 2 for CPU bound tests -# (SQLite) and 4 for I/O bound tests (PostgreSQL). -matrix: - fast_finish: true - include: - - name: "pep8" - python: 3.6 - env: TOX_ENV="pep8,check_isort,packaging" - - - name: "py2.7 / sqlite" - python: 2.7 - env: TOX_ENV=py27,codecov TRIAL_FLAGS="-j 2" - - - name: "py2.7 / sqlite / olddeps" - python: 2.7 - env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2" - - - name: "py2.7 / postgres9.5" - python: 2.7 - addons: - postgresql: "9.5" - env: TOX_ENV=py27-postgres,codecov TRIAL_FLAGS="-j 4" - services: - - postgresql - - - name: "py3.5 / sqlite" - python: 3.5 - env: TOX_ENV=py35,codecov TRIAL_FLAGS="-j 2" - - - name: "py3.7 / sqlite" - python: 3.7 - env: TOX_ENV=py37,codecov TRIAL_FLAGS="-j 2" - - - name: "py3.7 / postgres9.4" - python: 3.7 - addons: - postgresql: "9.4" - env: TOX_ENV=py37-postgres TRIAL_FLAGS="-j 4" - services: - - postgresql - - - name: "py3.7 / postgres9.5" - python: 3.7 - addons: - postgresql: "9.5" - env: TOX_ENV=py37-postgres,codecov TRIAL_FLAGS="-j 4" - services: - - postgresql - - - # we only need to check for the newsfragment if it's a PR build - if: type = pull_request - name: "check-newsfragment" - python: 3.6 - script: scripts-dev/check-newsfragment - -install: - # this just logs the postgres version we will be testing against (if any) - - psql -At -U postgres -c 'select version();' || true - - - pip install tox - - # if we don't have python3.6 in this environment, travis unhelpfully gives us - # a `python3.6` on our path which does nothing but spit out a warning. Tox - # tries to run it (even if we're not running a py36 env), so the build logs - # then have warnings which look like errors. To reduce the noise, remove the - # non-functional python3.6. - - ( ! command -v python3.6 || python3.6 --version ) &>/dev/null || rm -f $(command -v python3.6) - -script: - - tox -e $TOX_ENV diff --git a/MANIFEST.in b/MANIFEST.in index eb2de60f72..0500dd6b87 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -39,6 +39,7 @@ prune .circleci prune .coveragerc prune debian prune .codecov.yml +prune .buildkite exclude jenkins* recursive-exclude jenkins *.sh diff --git a/changelog.d/4752.misc b/changelog.d/4752.misc new file mode 100644 index 0000000000..fb1e76edce --- /dev/null +++ b/changelog.d/4752.misc @@ -0,0 +1 @@ +Change from TravisCI to Buildkite for CI. diff --git a/synapse/server.pyi b/synapse/server.pyi index 06cd083a74..fb8df56cd5 100644 --- a/synapse/server.pyi +++ b/synapse/server.pyi @@ -7,9 +7,9 @@ import synapse.handlers.auth import synapse.handlers.deactivate_account import synapse.handlers.device import synapse.handlers.e2e_keys +import synapse.handlers.message import synapse.handlers.room import synapse.handlers.room_member -import synapse.handlers.message import synapse.handlers.set_password import synapse.rest.media.v1.media_repository import synapse.server_notices.server_notices_manager diff --git a/tests/utils.py b/tests/utils.py index e8ab312528..cf49833a43 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -45,7 +45,9 @@ from synapse.util.ratelimitutils import FederationRateLimiter # set this to True to run the tests against postgres instead of sqlite. USE_POSTGRES_FOR_TESTS = os.environ.get("SYNAPSE_POSTGRES", False) LEAVE_DB = os.environ.get("SYNAPSE_LEAVE_DB", False) -POSTGRES_USER = os.environ.get("SYNAPSE_POSTGRES_USER", "postgres") +POSTGRES_USER = os.environ.get("SYNAPSE_POSTGRES_USER", None) +POSTGRES_HOST = os.environ.get("SYNAPSE_POSTGRES_HOST", None) +POSTGRES_PASSWORD = os.environ.get("SYNAPSE_POSTGRES_PASSWORD", None) POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),) @@ -58,6 +60,8 @@ def setupdb(): "args": { "database": POSTGRES_BASE_DB, "user": POSTGRES_USER, + "host": POSTGRES_HOST, + "password": POSTGRES_PASSWORD, "cp_min": 1, "cp_max": 5, }, @@ -66,7 +70,9 @@ def setupdb(): config.password_providers = [] config.database_config = pgconfig db_engine = create_engine(pgconfig) - db_conn = db_engine.module.connect(user=POSTGRES_USER) + db_conn = db_engine.module.connect( + user=POSTGRES_USER, host=POSTGRES_HOST, password=POSTGRES_PASSWORD + ) db_conn.autocommit = True cur = db_conn.cursor() cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,)) @@ -76,7 +82,10 @@ def setupdb(): # Set up in the db db_conn = db_engine.module.connect( - database=POSTGRES_BASE_DB, user=POSTGRES_USER + database=POSTGRES_BASE_DB, + user=POSTGRES_USER, + host=POSTGRES_HOST, + password=POSTGRES_PASSWORD, ) cur = db_conn.cursor() _get_or_create_schema_state(cur, db_engine) @@ -86,7 +95,9 @@ def setupdb(): db_conn.close() def _cleanup(): - db_conn = db_engine.module.connect(user=POSTGRES_USER) + db_conn = db_engine.module.connect( + user=POSTGRES_USER, host=POSTGRES_HOST, password=POSTGRES_PASSWORD + ) db_conn.autocommit = True cur = db_conn.cursor() cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,)) @@ -206,7 +217,14 @@ def setup_test_homeserver( config.database_config = { "name": "psycopg2", - "args": {"database": test_db, "cp_min": 1, "cp_max": 5}, + "args": { + "database": test_db, + "host": POSTGRES_HOST, + "password": POSTGRES_PASSWORD, + "user": POSTGRES_USER, + "cp_min": 1, + "cp_max": 5, + }, } else: config.database_config = { @@ -220,7 +238,10 @@ def setup_test_homeserver( # the template database we generate in setupdb() if datastore is None and isinstance(db_engine, PostgresEngine): db_conn = db_engine.module.connect( - database=POSTGRES_BASE_DB, user=POSTGRES_USER + database=POSTGRES_BASE_DB, + user=POSTGRES_USER, + host=POSTGRES_HOST, + password=POSTGRES_PASSWORD, ) db_conn.autocommit = True cur = db_conn.cursor() @@ -270,7 +291,10 @@ def setup_test_homeserver( # Drop the test database db_conn = db_engine.module.connect( - database=POSTGRES_BASE_DB, user=POSTGRES_USER + database=POSTGRES_BASE_DB, + user=POSTGRES_USER, + host=POSTGRES_HOST, + password=POSTGRES_PASSWORD, ) db_conn.autocommit = True cur = db_conn.cursor() @@ -492,7 +516,7 @@ class MockClock(object): return t def looping_call(self, function, interval): - self.loopers.append([function, interval / 1000., self.now]) + self.loopers.append([function, interval / 1000.0, self.now]) def cancel_call_later(self, timer, ignore_errs=False): if timer[2]: @@ -528,7 +552,7 @@ class MockClock(object): looped[2] = self.now def advance_time_msec(self, ms): - self.advance_time(ms / 1000.) + self.advance_time(ms / 1000.0) def time_bound_deferred(self, d, *args, **kwargs): # We don't bother timing things out for now. @@ -637,7 +661,7 @@ def create_room(hs, room_id, creator_id): "sender": creator_id, "room_id": room_id, "content": {}, - } + }, ) event, context = yield event_creation_handler.create_new_client_event(builder) -- cgit 1.5.1 From 68f47d6744ac2b4c6ac8b59b8c52a537a5072b4c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 27 Feb 2019 22:29:10 +0000 Subject: Fix parsing of Content-Disposition headers (#4763) * Fix parsing of Content-Disposition headers TIL: filenames in content-dispostion headers can contain semicolons, and aren't %-encoded. * fix python2 incompatibility * Fix docstrings --- changelog.d/4763.bugfix | 1 + synapse/rest/media/v1/_base.py | 85 ++++++++++++++++++++++++++++++---------- tests/rest/media/v1/test_base.py | 45 +++++++++++++++++++++ 3 files changed, 111 insertions(+), 20 deletions(-) create mode 100644 changelog.d/4763.bugfix create mode 100644 tests/rest/media/v1/test_base.py (limited to 'synapse') diff --git a/changelog.d/4763.bugfix b/changelog.d/4763.bugfix new file mode 100644 index 0000000000..213ea44b70 --- /dev/null +++ b/changelog.d/4763.bugfix @@ -0,0 +1 @@ +Fix parsing of Content-Disposition headers on remote media requests and URL previews. diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py index d16a30acd8..fece1ef0b8 100644 --- a/synapse/rest/media/v1/_base.py +++ b/synapse/rest/media/v1/_base.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2019 New Vector Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -213,8 +214,7 @@ def get_filename_from_headers(headers): Content-Disposition HTTP header. Args: - headers (twisted.web.http_headers.Headers): The HTTP - request headers. + headers (dict[bytes, list[bytes]]): The HTTP request headers. Returns: A Unicode string of the filename, or None. @@ -225,23 +225,12 @@ def get_filename_from_headers(headers): if not content_disposition[0]: return - # dict of unicode: bytes, corresponding to the key value sections of the - # Content-Disposition header. - params = {} - parts = content_disposition[0].split(b";") - for i in parts: - # Split into key-value pairs, if able - # We don't care about things like `inline`, so throw it out - if b"=" not in i: - continue - - key, value = i.strip().split(b"=") - params[key.decode('ascii')] = value + _, params = _parse_header(content_disposition[0]) upload_name = None # First check if there is a valid UTF-8 filename - upload_name_utf8 = params.get("filename*", None) + upload_name_utf8 = params.get(b"filename*", None) if upload_name_utf8: if upload_name_utf8.lower().startswith(b"utf-8''"): upload_name_utf8 = upload_name_utf8[7:] @@ -267,12 +256,68 @@ def get_filename_from_headers(headers): # If there isn't check for an ascii name. if not upload_name: - upload_name_ascii = params.get("filename", None) + upload_name_ascii = params.get(b"filename", None) if upload_name_ascii and is_ascii(upload_name_ascii): - # Make sure there's no %-quoted bytes. If there is, reject it as - # non-valid ASCII. - if b"%" not in upload_name_ascii: - upload_name = upload_name_ascii.decode('ascii') + upload_name = upload_name_ascii.decode('ascii') # This may be None here, indicating we did not find a matching name. return upload_name + + +def _parse_header(line): + """Parse a Content-type like header. + + Cargo-culted from `cgi`, but works on bytes rather than strings. + + Args: + line (bytes): header to be parsed + + Returns: + Tuple[bytes, dict[bytes, bytes]]: + the main content-type, followed by the parameter dictionary + """ + parts = _parseparam(b';' + line) + key = next(parts) + pdict = {} + for p in parts: + i = p.find(b'=') + if i >= 0: + name = p[:i].strip().lower() + value = p[i + 1:].strip() + + # strip double-quotes + if len(value) >= 2 and value[0:1] == value[-1:] == b'"': + value = value[1:-1] + value = value.replace(b'\\\\', b'\\').replace(b'\\"', b'"') + pdict[name] = value + + return key, pdict + + +def _parseparam(s): + """Generator which splits the input on ;, respecting double-quoted sequences + + Cargo-culted from `cgi`, but works on bytes rather than strings. + + Args: + s (bytes): header to be parsed + + Returns: + Iterable[bytes]: the split input + """ + while s[:1] == b';': + s = s[1:] + + # look for the next ; + end = s.find(b';') + + # if there is an odd number of " marks between here and the next ;, skip to the + # next ; instead + while end > 0 and (s.count(b'"', 0, end) - s.count(b'\\"', 0, end)) % 2: + end = s.find(b';', end + 1) + + if end < 0: + end = len(s) + f = s[:end] + yield f.strip() + s = s[end:] diff --git a/tests/rest/media/v1/test_base.py b/tests/rest/media/v1/test_base.py new file mode 100644 index 0000000000..af8f74eb42 --- /dev/null +++ b/tests/rest/media/v1/test_base.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.rest.media.v1._base import get_filename_from_headers + +from tests import unittest + + +class GetFileNameFromHeadersTests(unittest.TestCase): + # input -> expected result + TEST_CASES = { + b"inline; filename=abc.txt": u"abc.txt", + b'inline; filename="azerty"': u"azerty", + b'inline; filename="aze%20rty"': u"aze%20rty", + b'inline; filename="aze\"rty"': u'aze"rty', + b'inline; filename="azer;ty"': u"azer;ty", + + b"inline; filename*=utf-8''foo%C2%A3bar": u"foo£bar", + } + + def tests(self): + for hdr, expected in self.TEST_CASES.items(): + res = get_filename_from_headers( + { + b'Content-Disposition': [hdr], + }, + ) + self.assertEqual( + res, expected, + "expected output for %s to be %s but was %s" % ( + hdr, expected, res, + ) + ) -- cgit 1.5.1 From b131cc77dfd53c2f66cb6e2399a6b10024cf24cd Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 27 Feb 2019 14:35:47 -0800 Subject: Make 'event_id' a required parameter in federated state requests (#4741) * make 'event_id' a required parameter in federated state requests As per the spec: https://matrix.org/docs/spec/server_server/r0.1.1.html#id40 Signed-off-by: Joseph Weston * add changelog entry for bugfix Signed-off-by: Joseph Weston * Update server.py --- changelog.d/4740.bugfix | 1 + synapse/federation/transport/server.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4740.bugfix (limited to 'synapse') diff --git a/changelog.d/4740.bugfix b/changelog.d/4740.bugfix new file mode 100644 index 0000000000..f82bb4227a --- /dev/null +++ b/changelog.d/4740.bugfix @@ -0,0 +1 @@ +'event_id' is now a required parameter in federated state requests, as per the matrix spec. diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 5ba94be2ec..ebb81be377 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -393,7 +393,7 @@ class FederationStateServlet(BaseFederationServlet): return self.handler.on_context_state_request( origin, context, - parse_string_from_args(query, "event_id", None), + parse_string_from_args(query, "event_id", None, required=True), ) @@ -404,7 +404,7 @@ class FederationStateIdsServlet(BaseFederationServlet): return self.handler.on_state_ids_request( origin, room_id, - parse_string_from_args(query, "event_id", None), + parse_string_from_args(query, "event_id", None, required=True), ) -- cgit 1.5.1 From ac61b45a75304e90c05a5dba153900cfc0adb206 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 28 Feb 2019 16:24:01 +0000 Subject: Minor docstring fixes for MatrixFederationAgent (#4765) --- changelog.d/4765.misc | 1 + synapse/http/federation/matrix_federation_agent.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4765.misc (limited to 'synapse') diff --git a/changelog.d/4765.misc b/changelog.d/4765.misc new file mode 100644 index 0000000000..c273fd0cc4 --- /dev/null +++ b/changelog.d/4765.misc @@ -0,0 +1 @@ +Minor docstring fixes for MatrixFederationAgent. \ No newline at end of file diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 384d8a37a2..1334c630cc 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -68,9 +68,13 @@ class MatrixFederationAgent(object): TLS policy to use for fetching .well-known files. None to use a default (browser-like) implementation. - srv_resolver (SrvResolver|None): + _srv_resolver (SrvResolver|None): SRVResolver impl to use for looking up SRV records. None to use a default implementation. + + _well_known_cache (TTLCache|None): + TTLCache impl for storing cached well-known lookups. None to use a default + implementation. """ def __init__( -- cgit 1.5.1 From 9ac72d9543011a1b34fbcfa9a0973b57309c5c44 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 1 Mar 2019 10:55:44 +0000 Subject: 0.99.2 --- CHANGES.md | 6 ++++++ debian/changelog | 6 +++--- synapse/__init__.py | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/CHANGES.md b/CHANGES.md index 22684420a4..58028dc32f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +Synapse 0.99.2 (2019-03-01) +=========================== + +No significant changes. + + Synapse 0.99.2rc1 (2019-02-27) ============================== diff --git a/debian/changelog b/debian/changelog index 8a59c708d7..fd77ce13a2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,9 @@ -matrix-synapse-py3 (0.99.2rc1) stable; urgency=medium +matrix-synapse-py3 (0.99.2) stable; urgency=medium * Fix overwriting of config settings on upgrade. - * New synapse release 0.99.2rc1. + * New synapse release 0.99.2. - -- Synapse Packaging team Wed, 27 Feb 2019 10:45:58 +0000 + -- Synapse Packaging team Fri, 01 Mar 2019 10:55:08 +0000 matrix-synapse-py3 (0.99.1.1) stable; urgency=medium diff --git a/synapse/__init__.py b/synapse/__init__.py index 29b1fe4c03..25c10244d3 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.2rc1" +__version__ = "0.99.2" -- cgit 1.5.1 From 30649529398a2a57c52b6878a5753a5bf650cf25 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 1 Mar 2019 16:47:12 +0000 Subject: Fix incorrect log about not persisting duplicate state event. (#4776) We were logging this when it was not true. --- changelog.d/4776.bugfix | 1 + synapse/handlers/message.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4776.bugfix (limited to 'synapse') diff --git a/changelog.d/4776.bugfix b/changelog.d/4776.bugfix new file mode 100644 index 0000000000..ce3e6ce33c --- /dev/null +++ b/changelog.d/4776.bugfix @@ -0,0 +1 @@ +Fix incorrect log about not persisting duplicate state event. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3981fe69ce..c762b58902 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -436,10 +436,11 @@ class EventCreationHandler(object): if event.is_state(): prev_state = yield self.deduplicate_state_event(event, context) - logger.info( - "Not bothering to persist duplicate state event %s", event.event_id, - ) if prev_state is not None: + logger.info( + "Not bothering to persist state event %s duplicated by %s", + event.event_id, prev_state.event_id, + ) defer.returnValue(prev_state) yield self.handle_new_client_event( -- cgit 1.5.1