From ae19a7db8c5eab43858b24453bbbb352f8b6152a Mon Sep 17 00:00:00 2001 From: rkfg Date: Thu, 6 Dec 2018 13:32:05 +0300 Subject: Prevent crash on pagination. --- synapse/handlers/pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 43f81bd607..f2be6c1185 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -253,7 +253,7 @@ class PaginationHandler(object): ) state = None - if event_filter and event_filter.lazy_load_members(): + if event_filter and event_filter.lazy_load_members() and len(events) > 0: # TODO: remove redundant members # FIXME: we also care about invite targets etc. -- cgit 1.5.1 From 6d65659b62d1e338987a071d2f053cc3447e7ff5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 21 Feb 2019 17:50:30 +0000 Subject: Run push_receipts_to_remotes as background job (#4707) I suspect the CPU usage metrics for this are going to /dev/null at the moment. --- changelog.d/4707.misc | 1 + synapse/handlers/receipts.py | 68 ++++++++++++++++++++++---------------------- 2 files changed, 35 insertions(+), 34 deletions(-) create mode 100644 changelog.d/4707.misc (limited to 'synapse/handlers') diff --git a/changelog.d/4707.misc b/changelog.d/4707.misc new file mode 100644 index 0000000000..ef0772b9af --- /dev/null +++ b/changelog.d/4707.misc @@ -0,0 +1 @@ +Run push_receipts_to_remotes as background job. diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 4c2690ba26..696469732c 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -16,8 +16,8 @@ import logging from twisted.internet import defer +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import get_domain_from_id -from synapse.util import logcontext from ._base import BaseHandler @@ -59,7 +59,9 @@ class ReceiptsHandler(BaseHandler): if is_new: # fire off a process in the background to send the receipt to # remote servers - self._push_remotes([receipt]) + run_as_background_process( + 'push_receipts_to_remotes', self._push_remotes, receipt + ) @defer.inlineCallbacks def _received_remote_receipt(self, origin, content): @@ -125,44 +127,42 @@ class ReceiptsHandler(BaseHandler): defer.returnValue(True) - @logcontext.preserve_fn # caller should not yield on this @defer.inlineCallbacks - def _push_remotes(self, receipts): - """Given a list of receipts, works out which remote servers should be + def _push_remotes(self, receipt): + """Given a receipt, works out which remote servers should be poked and pokes them. """ try: - # TODO: Some of this stuff should be coallesced. - for receipt in receipts: - room_id = receipt["room_id"] - receipt_type = receipt["receipt_type"] - user_id = receipt["user_id"] - event_ids = receipt["event_ids"] - data = receipt["data"] - - users = yield self.state.get_current_user_in_room(room_id) - remotedomains = set(get_domain_from_id(u) for u in users) - remotedomains = remotedomains.copy() - remotedomains.discard(self.server_name) - - logger.debug("Sending receipt to: %r", remotedomains) - - for domain in remotedomains: - self.federation.send_edu( - destination=domain, - edu_type="m.receipt", - content={ - room_id: { - receipt_type: { - user_id: { - "event_ids": event_ids, - "data": data, - } + # TODO: optimise this to move some of the work to the workers. + room_id = receipt["room_id"] + receipt_type = receipt["receipt_type"] + user_id = receipt["user_id"] + event_ids = receipt["event_ids"] + data = receipt["data"] + + users = yield self.state.get_current_user_in_room(room_id) + remotedomains = set(get_domain_from_id(u) for u in users) + remotedomains = remotedomains.copy() + remotedomains.discard(self.server_name) + + logger.debug("Sending receipt to: %r", remotedomains) + + for domain in remotedomains: + self.federation.send_edu( + destination=domain, + edu_type="m.receipt", + content={ + room_id: { + receipt_type: { + user_id: { + "event_ids": event_ids, + "data": data, } - }, + } }, - key=(room_id, receipt_type, user_id), - ) + }, + key=(room_id, receipt_type, user_id), + ) except Exception: logger.exception("Error pushing receipts to remote servers") -- cgit 1.5.1 From 7b288826b76cae63b553fced10d3779355b592ca Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Feb 2019 11:33:51 +0000 Subject: Fix backfill storing incorrect state for events --- synapse/handlers/federation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 083f2e0ac3..40655bf92c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -772,8 +772,11 @@ class FederationHandler(BaseHandler): ev_infos = [] for a in auth_events.values(): - if a.event_id in seen_events: + # We only want to persist auth events as outliers that we haven't + # seen and aren't about to persist as part of the backfilled chunk. + if a.event_id in seen_events or a.event_id in event_map: continue + a.internal_metadata.outlier = True ev_infos.append({ "event": a, -- cgit 1.5.1 From 9342cc6ab167836ca3965923989c33f5230c27e9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 10:02:12 +0000 Subject: Add comments and paranoia --- synapse/handlers/federation.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 40655bf92c..fbf044b407 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -770,6 +770,18 @@ class FederationHandler(BaseHandler): set(auth_events.keys()) | set(state_events.keys()) ) + # We now have a chunk of events plus associated state and auth chain to + # persist. We do the persistence in two steps: + # 1. Auth events and state get persisted as outliers, plus the + # backward extremities get persisted (as non-outliers). + # 2. The rest of the events in the chunk get persisted one by one, as + # each one depends on the previous event for its state. + # + # The important thing is that events in the chunk get persisted as + # non-outliers, including when those events are also in the state or + # auth chain. Caution must therefore be taken to ensure that they are + # not accidentally marked as outliers. + ev_infos = [] for a in auth_events.values(): # We only want to persist auth events as outliers that we haven't @@ -789,13 +801,18 @@ class FederationHandler(BaseHandler): }) for e_id in events_to_state: + # For paranoia we ensure that these events are marked as + # non-outliers + ev = event_map[e_id] + ev.internal_metadata.outlier = False + ev_infos.append({ - "event": event_map[e_id], + "event": ev, "state": events_to_state[e_id], "auth_events": { (auth_events[a_id].type, auth_events[a_id].state_key): auth_events[a_id] - for a_id in event_map[e_id].auth_event_ids() + for a_id in ev.auth_event_ids() if a_id in auth_events } }) @@ -811,6 +828,10 @@ class FederationHandler(BaseHandler): if event in events_to_state: continue + # For paranoia we ensure that these events are marked as + # non-outliers + event.internal_metadata.outlier = False + # We store these one at a time since each event depends on the # previous to work out the state. # TODO: We can probably do something more clever here. -- cgit 1.5.1 From 890cb048fdbb05bcd8b80d3038f4811a1fdfe9f0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 14:42:11 +0000 Subject: Assert rather than clobber the values --- synapse/handlers/federation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index fbf044b407..8e6d0a3bbc 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -804,7 +804,7 @@ class FederationHandler(BaseHandler): # For paranoia we ensure that these events are marked as # non-outliers ev = event_map[e_id] - ev.internal_metadata.outlier = False + assert(not event.internal_metadata.is_outlier()) ev_infos.append({ "event": ev, @@ -830,7 +830,7 @@ class FederationHandler(BaseHandler): # For paranoia we ensure that these events are marked as # non-outliers - event.internal_metadata.outlier = False + assert(not event.internal_metadata.is_outlier()) # We store these one at a time since each event depends on the # previous to work out the state. -- cgit 1.5.1 From d730c2c22b53bef52afb6b26a878acc186e4ecae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 14:45:02 +0000 Subject: More comments --- synapse/handlers/federation.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8e6d0a3bbc..ca9b281be1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -782,6 +782,7 @@ class FederationHandler(BaseHandler): # auth chain. Caution must therefore be taken to ensure that they are # not accidentally marked as outliers. + # Step 1a: persist auth events that *don't* appear in the chunk ev_infos = [] for a in auth_events.values(): # We only want to persist auth events as outliers that we haven't @@ -800,6 +801,8 @@ class FederationHandler(BaseHandler): } }) + # Step 1b: persist the events in the chunk we fetched state for (i.e. + # the backwards extremities) as non-outliers. for e_id in events_to_state: # For paranoia we ensure that these events are marked as # non-outliers @@ -822,6 +825,7 @@ class FederationHandler(BaseHandler): backfilled=True, ) + # Step 2: Persist the rest of the events in the chunk one by one events.sort(key=lambda e: e.depth) for event in events: -- cgit 1.5.1 From e4b078a6004ea34684c27056825346178c22aa0f Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 22 Feb 2019 12:11:43 +0000 Subject: Config option to prevent showing non-fed rooms in fed /publicRooms --- synapse/config/room_directory.py | 11 +++++++++++ synapse/handlers/room_list.py | 17 +++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py index 9da13ab11b..08d1d26e58 100644 --- a/synapse/config/room_directory.py +++ b/synapse/config/room_directory.py @@ -27,6 +27,10 @@ class RoomDirectoryConfig(Config): for rule in alias_creation_rules ] + self.allow_non_federated_in_public_rooms = config.get( + "allow_non_federated_in_public_rooms", True, + ) + def default_config(self, config_dir_path, server_name, **kwargs): return """ # The `alias_creation` option controls who's allowed to create aliases @@ -42,6 +46,13 @@ class RoomDirectoryConfig(Config): - user_id: "*" alias: "*" action: allow + + # Specify whether rooms that only allow local users to join should be + # shown in the federation public room directory. + # + # Note that this does not affect the room directory shown to users on + # this homeserver, only those on other homeservers. + #allow_non_federated_in_public_rooms: True """ def is_alias_creation_allowed(self, user_id, alias): diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index dc88620885..47aeeb1d88 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -47,6 +47,7 @@ class RoomListHandler(BaseHandler): self.response_cache = ResponseCache(hs, "room_list") self.remote_response_cache = ResponseCache(hs, "remote_room_list", timeout_ms=30 * 1000) + self.config = hs.get_config() def get_local_public_room_list(self, limit=None, since_token=None, search_filter=None, @@ -286,14 +287,16 @@ class RoomListHandler(BaseHandler): # We've already got enough, so lets just drop it. return - result = yield self.generate_room_entry(room_id, num_joined_users) + result = yield self.generate_room_entry(room_id, num_joined_users, + allow_federated=self.config.allow_non_federated_in_public_rooms) if result and _matches_room_entry(result, search_filter): chunk.append(result) @cachedInlineCallbacks(num_args=1, cache_context=True) def generate_room_entry(self, room_id, num_joined_users, cache_context, - with_alias=True, allow_private=False): + with_alias=True, allow_private=False, + allow_federated=True): """Returns the entry for a room """ result = { @@ -308,6 +311,7 @@ class RoomListHandler(BaseHandler): event_map = yield self.store.get_events([ event_id for key, event_id in iteritems(current_state_ids) if key[0] in ( + EventTypes.Create, EventTypes.JoinRules, EventTypes.Name, EventTypes.Topic, @@ -324,12 +328,21 @@ class RoomListHandler(BaseHandler): } # Double check that this is actually a public room. + join_rules_event = current_state.get((EventTypes.JoinRules, "")) if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) if not allow_private and join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) + if not allow_federated: + # Disallow non-federated from appearing + create_event = current_state.get((EventTypes.Create, "")) + if create_event: + federate = create_event.content.get("m.federate", True) + if federate == False: + defer.returnValue(None) + if with_alias: aliases = yield self.store.get_aliases_for_room( room_id, on_invalidate=cache_context.invalidate -- cgit 1.5.1 From bd398b874eb375e92939a087a64258a4feb91f49 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 22 Feb 2019 15:43:11 +0000 Subject: Don't restrict non-fed rooms over client APIs --- synapse/federation/transport/server.py | 3 ++- synapse/groups/groups_server.py | 4 ++-- synapse/handlers/room_list.py | 29 ++++++++++++++++++----------- 3 files changed, 22 insertions(+), 14 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 7288d49074..6d4a26f595 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -697,7 +697,8 @@ class PublicRoomList(BaseFederationServlet): data = yield self.handler.get_local_public_room_list( limit, since_token, - network_tuple=network_tuple + network_tuple=network_tuple, + from_federation=True, ) defer.returnValue((200, data)) diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index 633c865ed8..691752a30c 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -113,7 +113,7 @@ class GroupsServerHandler(object): room_id = room_entry["room_id"] joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), + room_id, True, len(joined_users), with_alias=False, allow_private=True, ) entry = dict(entry) # so we don't change whats cached @@ -544,7 +544,7 @@ class GroupsServerHandler(object): joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), + room_id, True, len(joined_users), with_alias=False, allow_private=True, ) diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 47aeeb1d88..bde9ec744f 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -51,7 +51,8 @@ class RoomListHandler(BaseHandler): def get_local_public_room_list(self, limit=None, since_token=None, search_filter=None, - network_tuple=EMPTY_THIRD_PARTY_ID,): + network_tuple=EMPTY_THIRD_PARTY_ID, + from_federation=False): """Generate a local public room list. There are multiple different lists: the main one plus one per third @@ -82,13 +83,15 @@ class RoomListHandler(BaseHandler): return self.response_cache.wrap( key, self._get_public_room_list, - limit, since_token, network_tuple=network_tuple, + limit, since_token, + network_tuple=network_tuple, from_federation=from_federation, ) @defer.inlineCallbacks def _get_public_room_list(self, limit=None, since_token=None, search_filter=None, - network_tuple=EMPTY_THIRD_PARTY_ID,): + network_tuple=EMPTY_THIRD_PARTY_ID, + from_federation=False,): if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: @@ -208,7 +211,8 @@ class RoomListHandler(BaseHandler): yield concurrently_execute( lambda r: self._append_room_entry_to_chunk( r, rooms_to_num_joined[r], - chunk, limit, search_filter + chunk, limit, search_filter, + from_federation=from_federation, ), batch, 5, ) @@ -279,7 +283,7 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def _append_room_entry_to_chunk(self, room_id, num_joined_users, chunk, limit, - search_filter): + search_filter, from_federation=False): """Generate the entry for a room in the public room list and append it to the `chunk` if it matches the search filter """ @@ -287,16 +291,19 @@ class RoomListHandler(BaseHandler): # We've already got enough, so lets just drop it. return - result = yield self.generate_room_entry(room_id, num_joined_users, - allow_federated=self.config.allow_non_federated_in_public_rooms) + if from_federation: + result = yield self.generate_room_entry(room_id, + self.config.allow_non_federated_in_public_rooms, + num_joined_users) + else: + result = yield self.generate_room_entry(room_id, True, num_joined_users) if result and _matches_room_entry(result, search_filter): chunk.append(result) - @cachedInlineCallbacks(num_args=1, cache_context=True) - def generate_room_entry(self, room_id, num_joined_users, cache_context, - with_alias=True, allow_private=False, - allow_federated=True): + @cachedInlineCallbacks(num_args=2, cache_context=True) + def generate_room_entry(self, room_id, allow_federated, num_joined_users, + cache_context, with_alias=True, allow_private=False): """Returns the entry for a room """ result = { -- cgit 1.5.1 From 07493607a84665be006ec52f897eb904d923721c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 25 Feb 2019 12:42:30 +0000 Subject: Docs and arg name clarification --- synapse/handlers/room_list.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index bde9ec744f..0636e5b8eb 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -302,9 +302,22 @@ class RoomListHandler(BaseHandler): chunk.append(result) @cachedInlineCallbacks(num_args=2, cache_context=True) - def generate_room_entry(self, room_id, allow_federated, num_joined_users, + def generate_room_entry(self, room_id, allow_non_federated, num_joined_users, cache_context, with_alias=True, allow_private=False): """Returns the entry for a room + + Args: + room_id (str): The room's ID. + allow_non_federated (bool): Whether rooms with federation + disabled should be shown. + num_joined_users (int): Number of users in the room. + cache_context: Information for cached responses. + with_alias (bool): Whether to return the room's aliases in the result. + allow_private (bool): Whether invite-only rooms should be shown. + + Returns: + Deferred[dict|None]: Returns a room entry as a dictionary, or None if this + room was determined not to be shown publicly. """ result = { "room_id": room_id, @@ -342,7 +355,7 @@ class RoomListHandler(BaseHandler): if not allow_private and join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) - if not allow_federated: + if not allow_non_federated: # Disallow non-federated from appearing create_event = current_state.get((EventTypes.Create, "")) if create_event: -- cgit 1.5.1 From 84c0a20dfeb043f1113a6e213bc7488ef21a0432 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 25 Feb 2019 15:23:27 +0000 Subject: Simplify call to generate_room_entry --- synapse/handlers/room_list.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 810c624c2b..69b0bc3dd1 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -300,24 +300,24 @@ class RoomListHandler(BaseHandler): # We've already got enough, so lets just drop it. return - if from_federation: - result = yield self.generate_room_entry(room_id, - self.config.allow_non_federated_in_public_rooms, - num_joined_users) - else: - result = yield self.generate_room_entry(room_id, True, num_joined_users) - - if result and _matches_room_entry(result, search_filter): + result = yield self.generate_room_entry(room_id, + num_joined_users) + + if from_federation and not self.config.allow_non_federated_in_public_rooms: + if result["m.federate"] = False: + # This is a non-federating room and the config has chosen not + # to show these rooms to other servers + chunk.append(None) + else if result and _matches_room_entry(result, search_filter): chunk.append(result) @cachedInlineCallbacks(num_args=2, cache_context=True) - def generate_room_entry(self, room_id, allow_non_federated, num_joined_users, + def generate_room_entry(self, room_id, num_joined_users, cache_context, with_alias=True, allow_private=False): """Returns the entry for a room Args: room_id (str): The room's ID. - allow_non_federated (bool): Whether rooms with federation disabled should be shown. num_joined_users (int): Number of users in the room. cache_context: Information for cached responses. @@ -364,13 +364,9 @@ class RoomListHandler(BaseHandler): if not allow_private and join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) - if not allow_non_federated: - # Disallow non-federated from appearing - create_event = current_state.get((EventTypes.Create, "")) - if create_event: - federate = create_event.content.get("m.federate", True) - if federate == False: - defer.returnValue(None) + # Return whether this room is open to federation users or not + create_event = current_state.get((EventTypes.Create, "")) + result["m.federate"] = create_event.content.get("m.federate", True) if with_alias: aliases = yield self.store.get_aliases_for_room( -- cgit 1.5.1 From 8aaf7ffc4415e9f7da2d34448b29fb2d0cb80574 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 25 Feb 2019 15:27:17 +0000 Subject: syntax derp --- synapse/handlers/room_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 69b0bc3dd1..b7f450e719 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -304,7 +304,7 @@ class RoomListHandler(BaseHandler): num_joined_users) if from_federation and not self.config.allow_non_federated_in_public_rooms: - if result["m.federate"] = False: + if result["m.federate"] == False: # This is a non-federating room and the config has chosen not # to show these rooms to other servers chunk.append(None) -- cgit 1.5.1 From 9c598dddcbd75f60d422164215f3b078def604b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 16:32:02 +0000 Subject: Fix typo --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ca9b281be1..f80486102a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -807,7 +807,7 @@ class FederationHandler(BaseHandler): # For paranoia we ensure that these events are marked as # non-outliers ev = event_map[e_id] - assert(not event.internal_metadata.is_outlier()) + assert(not ev.internal_metadata.is_outlier()) ev_infos.append({ "event": ev, -- cgit 1.5.1 From c7b333c545adf8958ec97771e6816ceb5e83f524 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 16:56:41 +0000 Subject: Log tracebacks correctly --- synapse/federation/federation_server.py | 5 +++-- synapse/handlers/pagination.py | 6 +++++- synapse/http/server.py | 8 ++++---- 3 files changed, 12 insertions(+), 7 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 3da86d4ba6..8f45c2badf 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -239,8 +239,9 @@ class FederationServer(FederationBase): f = failure.Failure() pdu_results[event_id] = {"error": str(e)} logger.error( - "Failed to handle PDU %s: %s", - event_id, f.getTraceback().rstrip(), + "Failed to handle PDU %s", + event_id, + exc_info=(f.type, f.value, f.getTracebackObject()), ) yield concurrently_execute( diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 084c1503da..e4fdae9266 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -136,7 +136,11 @@ class PaginationHandler(object): logger.info("[purge] complete") self._purges_by_id[purge_id].status = PurgeStatus.STATUS_COMPLETE except Exception: - logger.error("[purge] failed: %s", Failure().getTraceback().rstrip()) + f = Failure() + logger.error( + "[purge] failed", + exc_info=(f.type, f.value, f.getTracebackObject()), + ) self._purges_by_id[purge_id].status = PurgeStatus.STATUS_FAILED finally: self._purges_in_progress_by_room.discard(room_id) diff --git a/synapse/http/server.py b/synapse/http/server.py index 6c67a25a11..16fb7935da 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -169,18 +169,18 @@ def _return_html_error(f, request): ) else: logger.error( - "Failed handle request %r: %s", + "Failed handle request %r", request, - f.getTraceback().rstrip(), + exc_info=(f.type, f.value, f.getTracebackObject()), ) else: code = http_client.INTERNAL_SERVER_ERROR msg = "Internal server error" logger.error( - "Failed handle request %r: %s", + "Failed handle request %r", request, - f.getTraceback().rstrip(), + exc_info=(f.type, f.value, f.getTracebackObject()), ) body = HTML_ERROR_TEMPLATE.format( -- cgit 1.5.1 From 1330aa4a8fdff5f6d1ff4bc5d27de674dfcd67e7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 25 Feb 2019 17:28:19 +0000 Subject: elif not else if --- synapse/handlers/room_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index b7f450e719..94cabc2a2e 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -308,7 +308,7 @@ class RoomListHandler(BaseHandler): # This is a non-federating room and the config has chosen not # to show these rooms to other servers chunk.append(None) - else if result and _matches_room_entry(result, search_filter): + elif result and _matches_room_entry(result, search_filter): chunk.append(result) @cachedInlineCallbacks(num_args=2, cache_context=True) -- cgit 1.5.1 From 70ea2f4e1df41458532c7964f4f707e04810e619 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 25 Feb 2019 19:15:36 +0000 Subject: switch from google.com to recaptcha.net for reCAPTCHA (#4731) * add trivial clarification about jemalloc * switch from google.com to recaptcha.net because https://developers.google.com/recaptcha/docs/faq#can-i-use-recaptcha-globally --- README.rst | 2 ++ synapse/config/captcha.py | 2 +- synapse/handlers/register.py | 4 ++-- synapse/rest/client/v2_alpha/auth.py | 2 +- synapse/static/client/register/index.html | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) (limited to 'synapse/handlers') diff --git a/README.rst b/README.rst index 9a7c04b55e..8e22109973 100644 --- a/README.rst +++ b/README.rst @@ -199,6 +199,8 @@ by installing the ``libjemalloc1`` package and adding this line to LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 +This can make a significant difference on Python 2.7 - it's unclear how +much of an improvement it provides on Python 3.x. Upgrading an existing Synapse ============================= diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 4064891ffb..d25196be08 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -47,5 +47,5 @@ class CaptchaConfig(Config): #captcha_bypass_secret: "YOUR_SECRET_HERE" # The API endpoint to use for verifying m.login.recaptcha responses. - recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" + recaptcha_siteverify_api: "https://www.recaptcha.net/recaptcha/api/siteverify" """ diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 24a4cb5a83..c0e06929bd 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -460,7 +460,7 @@ class RegistrationHandler(BaseHandler): lines = response.split('\n') json = { "valid": lines[0] == 'true', - "error_url": "http://www.google.com/recaptcha/api/challenge?" + + "error_url": "http://www.recaptcha.net/recaptcha/api/challenge?" + "error=%s" % lines[1] } defer.returnValue(json) @@ -471,7 +471,7 @@ class RegistrationHandler(BaseHandler): Used only by c/s api v1 """ data = yield self.captcha_client.post_urlencoded_get_raw( - "http://www.google.com:80/recaptcha/api/verify", + "http://www.recaptcha.net:80/recaptcha/api/verify", args={ 'privatekey': private_key, 'remoteip': ip_addr, diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index f7bb710642..ac035c7735 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -33,7 +33,7 @@ RECAPTCHA_TEMPLATE = """ Authentication - diff --git a/synapse/static/client/register/index.html b/synapse/static/client/register/index.html index 886f2edd1f..6edc4deb03 100644 --- a/synapse/static/client/register/index.html +++ b/synapse/static/client/register/index.html @@ -4,7 +4,7 @@ - + -- cgit 1.5.1 From 71669a0fba524e4d8cdb9e0076dca5d2770ef788 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 11:25:00 +0000 Subject: Address rich comments --- synapse/handlers/room_list.py | 36 ++++++++++++++++++++++++++++++------ synapse/server.pyi | 2 +- 2 files changed, 31 insertions(+), 7 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 94cabc2a2e..a9725ade37 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -97,7 +97,23 @@ class RoomListHandler(BaseHandler): def _get_public_room_list(self, limit=None, since_token=None, search_filter=None, network_tuple=EMPTY_THIRD_PARTY_ID, - from_federation=False, timeout=None,): + from_federation=False, + timeout=None,): + """Generate a public room list. + + Args: + limit (int): Maximum amount of rooms to return. + since_token (str) + search_filter (dict): Dictionary to filter rooms by. + network_tuple (ThirdPartyInstanceID): Which public list to use. + This can be (None, None) to indicate the main list, or a particular + appservice and network id to use an appservice specific one. + Setting to None returns all public rooms across all lists. + from_federation (bool): Whether this request originated from a + federating server or a client. Used for room filtering. + timeout (int): Amount of seconds to wait for a response before + timing out. + """ if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: @@ -295,19 +311,28 @@ class RoomListHandler(BaseHandler): search_filter, from_federation=False): """Generate the entry for a room in the public room list and append it to the `chunk` if it matches the search filter + + Args: + room_id (str): The ID of the room. + num_joined_users (int): The number of joined users in the room. + chunk (list) + limit (int): Maximum amount of rooms to display. Function will + return if length of chunk is greater than limit + 1. + search_filter (dict) + from_federation (bool): Whether this request originated from a + federating server or a client. Used for room filtering. """ if limit and len(chunk) > limit + 1: # We've already got enough, so lets just drop it. return - result = yield self.generate_room_entry(room_id, - num_joined_users) + result = yield self.generate_room_entry(room_id, num_joined_users) if from_federation and not self.config.allow_non_federated_in_public_rooms: - if result["m.federate"] == False: + if result["m.federate"] is False: # This is a non-federating room and the config has chosen not # to show these rooms to other servers - chunk.append(None) + return elif result and _matches_room_entry(result, search_filter): chunk.append(result) @@ -318,7 +343,6 @@ class RoomListHandler(BaseHandler): Args: room_id (str): The room's ID. - disabled should be shown. num_joined_users (int): Number of users in the room. cache_context: Information for cached responses. with_alias (bool): Whether to return the room's aliases in the result. diff --git a/synapse/server.pyi b/synapse/server.pyi index 06cd083a74..fb8df56cd5 100644 --- a/synapse/server.pyi +++ b/synapse/server.pyi @@ -7,9 +7,9 @@ import synapse.handlers.auth import synapse.handlers.deactivate_account import synapse.handlers.device import synapse.handlers.e2e_keys +import synapse.handlers.message import synapse.handlers.room import synapse.handlers.room_member -import synapse.handlers.message import synapse.handlers.set_password import synapse.rest.media.v1.media_repository import synapse.server_notices.server_notices_manager -- cgit 1.5.1 From 6946c20111cb7c460deb340b61236dce9f4fd878 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 11:27:19 +0000 Subject: Result may be None --- synapse/handlers/room_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index a9725ade37..7c21f142a0 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -329,7 +329,7 @@ class RoomListHandler(BaseHandler): result = yield self.generate_room_entry(room_id, num_joined_users) if from_federation and not self.config.allow_non_federated_in_public_rooms: - if result["m.federate"] is False: + if not result or result["m.federate"] is False: # This is a non-federating room and the config has chosen not # to show these rooms to other servers return -- cgit 1.5.1 From 6728bf39405c52e4f2473c1e19a7648134361c15 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 11:52:52 +0000 Subject: Make not showing non-federated rooms the default --- synapse/config/room_directory.py | 12 ------------ synapse/handlers/room_list.py | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py index 3322cf2eea..9b897abe3c 100644 --- a/synapse/config/room_directory.py +++ b/synapse/config/room_directory.py @@ -52,10 +52,6 @@ class RoomDirectoryConfig(Config): ) ] - self.allow_non_federated_in_public_rooms = config.get( - "allow_non_federated_in_public_rooms", True, - ) - def default_config(self, config_dir_path, server_name, **kwargs): return """ # The `alias_creation` option controls who's allowed to create aliases @@ -114,14 +110,6 @@ class RoomDirectoryConfig(Config): # alias: "*" # room_id: "*" # action: allow - - # Specify whether rooms that only allow local users to join should be - # shown in the federation public room directory. - # - # Note that this does not affect the room directory shown to users on - # this homeserver, only those on other homeservers. - # - #allow_non_federated_in_public_rooms: True """ def is_alias_creation_allowed(self, user_id, room_id, alias): diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 7c21f142a0..f1a51a7ca9 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -328,7 +328,7 @@ class RoomListHandler(BaseHandler): result = yield self.generate_room_entry(room_id, num_joined_users) - if from_federation and not self.config.allow_non_federated_in_public_rooms: + if from_federation: if not result or result["m.federate"] is False: # This is a non-federating room and the config has chosen not # to show these rooms to other servers -- cgit 1.5.1 From 40c2271680501c5ca2c29349d62df42d80b953db Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 12:04:34 +0000 Subject: Clean up room chunk logic --- synapse/handlers/room_list.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index f1a51a7ca9..e6038a1069 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -327,13 +327,15 @@ class RoomListHandler(BaseHandler): return result = yield self.generate_room_entry(room_id, num_joined_users) + if not result: + return - if from_federation: - if not result or result["m.federate"] is False: - # This is a non-federating room and the config has chosen not - # to show these rooms to other servers - return - elif result and _matches_room_entry(result, search_filter): + if from_federation and result["m.federate"] is False: + # This is a room that other servers cannot join. Do not show them + # this room. + return + + if _matches_room_entry(result, search_filter): chunk.append(result) @cachedInlineCallbacks(num_args=2, cache_context=True) -- cgit 1.5.1 From 16565e67dbc81ddf5fc16d39f82accbd508ac13f Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 12:12:48 +0000 Subject: Correct docstring types and chunk logic --- synapse/handlers/room_list.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index e6038a1069..7ab8476680 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -59,9 +59,9 @@ class RoomListHandler(BaseHandler): party network. A client can ask for a specific list or to return all. Args: - limit (int) - since_token (str) - search_filter (dict) + limit (int|None) + since_token (str|None) + search_filter (dict|None) network_tuple (ThirdPartyInstanceID): Which public list to use. This can be (None, None) to indicate the main list, or a particular appservice and network id to use an appservice specific one. @@ -102,16 +102,16 @@ class RoomListHandler(BaseHandler): """Generate a public room list. Args: - limit (int): Maximum amount of rooms to return. - since_token (str) - search_filter (dict): Dictionary to filter rooms by. + limit (int|None): Maximum amount of rooms to return. + since_token (str|None) + search_filter (dict|None): Dictionary to filter rooms by. network_tuple (ThirdPartyInstanceID): Which public list to use. This can be (None, None) to indicate the main list, or a particular appservice and network id to use an appservice specific one. Setting to None returns all public rooms across all lists. from_federation (bool): Whether this request originated from a federating server or a client. Used for room filtering. - timeout (int): Amount of seconds to wait for a response before + timeout (int|None): Amount of seconds to wait for a response before timing out. """ if since_token and since_token != "END": @@ -316,9 +316,9 @@ class RoomListHandler(BaseHandler): room_id (str): The ID of the room. num_joined_users (int): The number of joined users in the room. chunk (list) - limit (int): Maximum amount of rooms to display. Function will + limit (int|None): Maximum amount of rooms to display. Function will return if length of chunk is greater than limit + 1. - search_filter (dict) + search_filter (dict|None) from_federation (bool): Whether this request originated from a federating server or a client. Used for room filtering. """ @@ -330,7 +330,8 @@ class RoomListHandler(BaseHandler): if not result: return - if from_federation and result["m.federate"] is False: + if from_federation: + if "m.federate" in result and not result["m.federate"]: # This is a room that other servers cannot join. Do not show them # this room. return -- cgit 1.5.1 From a712aa3a9c5c58eca6ecfa0cb23636035fbbf449 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 12:13:55 +0000 Subject: Correct indent --- synapse/handlers/room_list.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 7ab8476680..0ce274e9ae 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -332,9 +332,9 @@ class RoomListHandler(BaseHandler): if from_federation: if "m.federate" in result and not result["m.federate"]: - # This is a room that other servers cannot join. Do not show them - # this room. - return + # This is a room that other servers cannot join. Do not show them + # this room. + return if _matches_room_entry(result, search_filter): chunk.append(result) -- cgit 1.5.1 From c4414768af5e2213b1ad2b16bc8e1b8062fd1e49 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 12:22:34 +0000 Subject: Cleaner chunk logic --- synapse/handlers/room_list.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 0ce274e9ae..c02fedcfe4 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -330,11 +330,10 @@ class RoomListHandler(BaseHandler): if not result: return - if from_federation: - if "m.federate" in result and not result["m.federate"]: - # This is a room that other servers cannot join. Do not show them - # this room. - return + if from_federation and not result.get("m.federate", True): + # This is a room that other servers cannot join. Do not show them + # this room. + return if _matches_room_entry(result, search_filter): chunk.append(result) -- cgit 1.5.1 From c74624a633ae883474a5ec3137bddd009165144c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 26 Feb 2019 13:20:38 +0000 Subject: Revert "Prevent showing non-fed rooms in fed /publicRooms" --- synapse/federation/transport/server.py | 3 +- synapse/groups/groups_server.py | 6 ++- synapse/handlers/room_list.py | 76 +++++----------------------------- synapse/server.pyi | 2 +- 4 files changed, 17 insertions(+), 70 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 5ba94be2ec..a2396ab466 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -736,8 +736,7 @@ class PublicRoomList(BaseFederationServlet): data = yield self.handler.get_local_public_room_list( limit, since_token, - network_tuple=network_tuple, - from_federation=True, + network_tuple=network_tuple ) defer.returnValue((200, data)) diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index a7eaead56b..633c865ed8 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -113,7 +113,8 @@ class GroupsServerHandler(object): room_id = room_entry["room_id"] joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), with_alias=False, allow_private=True, + room_id, len(joined_users), + with_alias=False, allow_private=True, ) entry = dict(entry) # so we don't change whats cached entry.pop("room_id", None) @@ -543,7 +544,8 @@ class GroupsServerHandler(object): joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), with_alias=False, allow_private=True, + room_id, len(joined_users), + with_alias=False, allow_private=True, ) if not entry: diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index c02fedcfe4..13e212d669 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -47,21 +47,19 @@ class RoomListHandler(BaseHandler): self.response_cache = ResponseCache(hs, "room_list") self.remote_response_cache = ResponseCache(hs, "remote_room_list", timeout_ms=30 * 1000) - self.config = hs.get_config() def get_local_public_room_list(self, limit=None, since_token=None, search_filter=None, - network_tuple=EMPTY_THIRD_PARTY_ID, - from_federation=False): + network_tuple=EMPTY_THIRD_PARTY_ID,): """Generate a local public room list. There are multiple different lists: the main one plus one per third party network. A client can ask for a specific list or to return all. Args: - limit (int|None) - since_token (str|None) - search_filter (dict|None) + limit (int) + since_token (str) + search_filter (dict) network_tuple (ThirdPartyInstanceID): Which public list to use. This can be (None, None) to indicate the main list, or a particular appservice and network id to use an appservice specific one. @@ -89,31 +87,14 @@ class RoomListHandler(BaseHandler): return self.response_cache.wrap( key, self._get_public_room_list, - limit, since_token, - network_tuple=network_tuple, from_federation=from_federation, + limit, since_token, network_tuple=network_tuple, ) @defer.inlineCallbacks def _get_public_room_list(self, limit=None, since_token=None, search_filter=None, network_tuple=EMPTY_THIRD_PARTY_ID, - from_federation=False, timeout=None,): - """Generate a public room list. - - Args: - limit (int|None): Maximum amount of rooms to return. - since_token (str|None) - search_filter (dict|None): Dictionary to filter rooms by. - network_tuple (ThirdPartyInstanceID): Which public list to use. - This can be (None, None) to indicate the main list, or a particular - appservice and network id to use an appservice specific one. - Setting to None returns all public rooms across all lists. - from_federation (bool): Whether this request originated from a - federating server or a client. Used for room filtering. - timeout (int|None): Amount of seconds to wait for a response before - timing out. - """ if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: @@ -236,8 +217,7 @@ class RoomListHandler(BaseHandler): yield concurrently_execute( lambda r: self._append_room_entry_to_chunk( r, rooms_to_num_joined[r], - chunk, limit, search_filter, - from_federation=from_federation, + chunk, limit, search_filter ), batch, 5, ) @@ -308,51 +288,23 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def _append_room_entry_to_chunk(self, room_id, num_joined_users, chunk, limit, - search_filter, from_federation=False): + search_filter): """Generate the entry for a room in the public room list and append it to the `chunk` if it matches the search filter - - Args: - room_id (str): The ID of the room. - num_joined_users (int): The number of joined users in the room. - chunk (list) - limit (int|None): Maximum amount of rooms to display. Function will - return if length of chunk is greater than limit + 1. - search_filter (dict|None) - from_federation (bool): Whether this request originated from a - federating server or a client. Used for room filtering. """ if limit and len(chunk) > limit + 1: # We've already got enough, so lets just drop it. return result = yield self.generate_room_entry(room_id, num_joined_users) - if not result: - return - - if from_federation and not result.get("m.federate", True): - # This is a room that other servers cannot join. Do not show them - # this room. - return - if _matches_room_entry(result, search_filter): + if result and _matches_room_entry(result, search_filter): chunk.append(result) - @cachedInlineCallbacks(num_args=2, cache_context=True) - def generate_room_entry(self, room_id, num_joined_users, - cache_context, with_alias=True, allow_private=False): + @cachedInlineCallbacks(num_args=1, cache_context=True) + def generate_room_entry(self, room_id, num_joined_users, cache_context, + with_alias=True, allow_private=False): """Returns the entry for a room - - Args: - room_id (str): The room's ID. - num_joined_users (int): Number of users in the room. - cache_context: Information for cached responses. - with_alias (bool): Whether to return the room's aliases in the result. - allow_private (bool): Whether invite-only rooms should be shown. - - Returns: - Deferred[dict|None]: Returns a room entry as a dictionary, or None if this - room was determined not to be shown publicly. """ result = { "room_id": room_id, @@ -366,7 +318,6 @@ class RoomListHandler(BaseHandler): event_map = yield self.store.get_events([ event_id for key, event_id in iteritems(current_state_ids) if key[0] in ( - EventTypes.Create, EventTypes.JoinRules, EventTypes.Name, EventTypes.Topic, @@ -383,17 +334,12 @@ class RoomListHandler(BaseHandler): } # Double check that this is actually a public room. - join_rules_event = current_state.get((EventTypes.JoinRules, "")) if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) if not allow_private and join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) - # Return whether this room is open to federation users or not - create_event = current_state.get((EventTypes.Create, "")) - result["m.federate"] = create_event.content.get("m.federate", True) - if with_alias: aliases = yield self.store.get_aliases_for_room( room_id, on_invalidate=cache_context.invalidate diff --git a/synapse/server.pyi b/synapse/server.pyi index fb8df56cd5..06cd083a74 100644 --- a/synapse/server.pyi +++ b/synapse/server.pyi @@ -7,9 +7,9 @@ import synapse.handlers.auth import synapse.handlers.deactivate_account import synapse.handlers.device import synapse.handlers.e2e_keys -import synapse.handlers.message import synapse.handlers.room import synapse.handlers.room_member +import synapse.handlers.message import synapse.handlers.set_password import synapse.rest.media.v1.media_repository import synapse.server_notices.server_notices_manager -- cgit 1.5.1 From 7a4632af9c4a26604b9d30f69eb81cb4630a82e7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 22 Feb 2019 12:11:43 +0000 Subject: Prevent showing non-fed rooms in fed /publicRooms --- synapse/federation/transport/server.py | 3 +- synapse/groups/groups_server.py | 6 ++-- synapse/handlers/room_list.py | 62 +++++++++++++++++++++++++++------- 3 files changed, 54 insertions(+), 17 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 7288d49074..6d4a26f595 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -697,7 +697,8 @@ class PublicRoomList(BaseFederationServlet): data = yield self.handler.get_local_public_room_list( limit, since_token, - network_tuple=network_tuple + network_tuple=network_tuple, + from_federation=True, ) defer.returnValue((200, data)) diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index 633c865ed8..a7eaead56b 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -113,8 +113,7 @@ class GroupsServerHandler(object): room_id = room_entry["room_id"] joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), - with_alias=False, allow_private=True, + room_id, len(joined_users), with_alias=False, allow_private=True, ) entry = dict(entry) # so we don't change whats cached entry.pop("room_id", None) @@ -544,8 +543,7 @@ class GroupsServerHandler(object): joined_users = yield self.store.get_users_in_room(room_id) entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), - with_alias=False, allow_private=True, + room_id, len(joined_users), with_alias=False, allow_private=True, ) if not entry: diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index dc88620885..b722e2175f 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -50,16 +50,17 @@ class RoomListHandler(BaseHandler): def get_local_public_room_list(self, limit=None, since_token=None, search_filter=None, - network_tuple=EMPTY_THIRD_PARTY_ID,): + network_tuple=EMPTY_THIRD_PARTY_ID, + from_federation=False): """Generate a local public room list. There are multiple different lists: the main one plus one per third party network. A client can ask for a specific list or to return all. Args: - limit (int) - since_token (str) - search_filter (dict) + limit (int|None) + since_token (str|None) + search_filter (dict|None) network_tuple (ThirdPartyInstanceID): Which public list to use. This can be (None, None) to indicate the main list, or a particular appservice and network id to use an appservice specific one. @@ -81,13 +82,15 @@ class RoomListHandler(BaseHandler): return self.response_cache.wrap( key, self._get_public_room_list, - limit, since_token, network_tuple=network_tuple, + limit, since_token, + network_tuple=network_tuple, from_federation=from_federation, ) @defer.inlineCallbacks def _get_public_room_list(self, limit=None, since_token=None, search_filter=None, - network_tuple=EMPTY_THIRD_PARTY_ID,): + network_tuple=EMPTY_THIRD_PARTY_ID, + from_federation=False,): if since_token and since_token != "END": since_token = RoomListNextBatch.from_token(since_token) else: @@ -207,7 +210,8 @@ class RoomListHandler(BaseHandler): yield concurrently_execute( lambda r: self._append_room_entry_to_chunk( r, rooms_to_num_joined[r], - chunk, limit, search_filter + chunk, limit, search_filter, + from_federation=from_federation, ), batch, 5, ) @@ -278,23 +282,51 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def _append_room_entry_to_chunk(self, room_id, num_joined_users, chunk, limit, - search_filter): + search_filter, from_federation=False): """Generate the entry for a room in the public room list and append it to the `chunk` if it matches the search filter + + Args: + room_id (str): The ID of the room. + num_joined_users (int): The number of joined users in the room. + chunk (list) + limit (int|None): Maximum amount of rooms to display. Function will + return if length of chunk is greater than limit + 1. + search_filter (dict|None) + from_federation (bool): Whether this request originated from a + federating server or a client. Used for room filtering. """ if limit and len(chunk) > limit + 1: # We've already got enough, so lets just drop it. return result = yield self.generate_room_entry(room_id, num_joined_users) + if not result: + return + + if from_federation and not result.get("m.federate", True): + # This is a room that other servers cannot join. Do not show them + # this room. + return - if result and _matches_room_entry(result, search_filter): + if _matches_room_entry(result, search_filter): chunk.append(result) - @cachedInlineCallbacks(num_args=1, cache_context=True) - def generate_room_entry(self, room_id, num_joined_users, cache_context, - with_alias=True, allow_private=False): + @cachedInlineCallbacks(num_args=2, cache_context=True) + def generate_room_entry(self, room_id, num_joined_users, + cache_context, with_alias=True, allow_private=False): """Returns the entry for a room + + Args: + room_id (str): The room's ID. + num_joined_users (int): Number of users in the room. + cache_context: Information for cached responses. + with_alias (bool): Whether to return the room's aliases in the result. + allow_private (bool): Whether invite-only rooms should be shown. + + Returns: + Deferred[dict|None]: Returns a room entry as a dictionary, or None if this + room was determined not to be shown publicly. """ result = { "room_id": room_id, @@ -308,6 +340,7 @@ class RoomListHandler(BaseHandler): event_map = yield self.store.get_events([ event_id for key, event_id in iteritems(current_state_ids) if key[0] in ( + EventTypes.Create, EventTypes.JoinRules, EventTypes.Name, EventTypes.Topic, @@ -324,12 +357,17 @@ class RoomListHandler(BaseHandler): } # Double check that this is actually a public room. + join_rules_event = current_state.get((EventTypes.JoinRules, "")) if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) if not allow_private and join_rule and join_rule != JoinRules.PUBLIC: defer.returnValue(None) + # Return whether this room is open to federation users or not + create_event = current_state.get((EventTypes.Create, "")) + result["m.federate"] = create_event.content.get("m.federate", True) + if with_alias: aliases = yield self.store.get_aliases_for_room( room_id, on_invalidate=cache_context.invalidate -- cgit 1.5.1 From 6fcb25202f34348e3411460496296252a2491bce Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Feb 2019 14:13:38 +0000 Subject: Put function def back to the way it was --- synapse/handlers/room_list.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index b722e2175f..4f51a464e7 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -312,9 +312,9 @@ class RoomListHandler(BaseHandler): if _matches_room_entry(result, search_filter): chunk.append(result) - @cachedInlineCallbacks(num_args=2, cache_context=True) - def generate_room_entry(self, room_id, num_joined_users, - cache_context, with_alias=True, allow_private=False): + @cachedInlineCallbacks(num_args=1, cache_context=True) + def generate_room_entry(self, room_id, num_joined_users, cache_context, + with_alias=True, allow_private=False): """Returns the entry for a room Args: -- cgit 1.5.1 From 30649529398a2a57c52b6878a5753a5bf650cf25 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 1 Mar 2019 16:47:12 +0000 Subject: Fix incorrect log about not persisting duplicate state event. (#4776) We were logging this when it was not true. --- changelog.d/4776.bugfix | 1 + synapse/handlers/message.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4776.bugfix (limited to 'synapse/handlers') diff --git a/changelog.d/4776.bugfix b/changelog.d/4776.bugfix new file mode 100644 index 0000000000..ce3e6ce33c --- /dev/null +++ b/changelog.d/4776.bugfix @@ -0,0 +1 @@ +Fix incorrect log about not persisting duplicate state event. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3981fe69ce..c762b58902 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -436,10 +436,11 @@ class EventCreationHandler(object): if event.is_state(): prev_state = yield self.deduplicate_state_event(event, context) - logger.info( - "Not bothering to persist duplicate state event %s", event.event_id, - ) if prev_state is not None: + logger.info( + "Not bothering to persist state event %s duplicated by %s", + event.event_id, prev_state.event_id, + ) defer.returnValue(prev_state) yield self.handle_new_client_event( -- cgit 1.5.1