From fcb05b4c8269ac7cb2e153e0a68c040a965d8279 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Jan 2016 17:37:09 +0000 Subject: Temporarily disable notification branch --- synapse/handlers/federation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'synapse/handlers/federation.py') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5dee24e21d..2f6359c768 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -36,7 +36,7 @@ from synapse.events.utils import prune_event from synapse.util.retryutils import NotRetryingDestination -from synapse.push.action_generator import ActionGenerator +# from synapse.push.action_generator import ActionGenerator from twisted.internet import defer @@ -244,11 +244,12 @@ class FederationHandler(BaseHandler): user = UserID.from_string(event.state_key) yield user_joined_room(self.distributor, user, event.room_id) - if not backfilled and not event.internal_metadata.is_outlier(): - action_generator = ActionGenerator(self.store) - yield action_generator.handle_push_actions_for_event( - event, self - ) + # Temporarily disable notifications due to performance concerns. + # if not backfilled and not event.internal_metadata.is_outlier(): + # action_generator = ActionGenerator(self.store) + # yield action_generator.handle_push_actions_for_event( + # event, self + # ) @defer.inlineCallbacks def _filter_events_for_server(self, server_name, room_id, events): -- cgit 1.5.1 From ac5a4477adc772e4416c868e8b16ae41a2c0c4ef Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 15 Jan 2016 16:27:26 +0000 Subject: Require unbanning before other membership changes --- synapse/api/errors.py | 1 + synapse/handlers/federation.py | 4 +-- synapse/handlers/message.py | 57 +++++++++++++++++++++++++++++++++--------- synapse/handlers/room.py | 55 ++++++++++++++++++++++++++++++++++++++-- synapse/rest/client/v1/room.py | 51 +++++++++---------------------------- tests/handlers/test_room.py | 6 ++--- 6 files changed, 116 insertions(+), 58 deletions(-) (limited to 'synapse/handlers/federation.py') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index ce0fc53668..b106fbed6d 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -29,6 +29,7 @@ class Codes(object): USER_IN_USE = "M_USER_IN_USE" ROOM_IN_USE = "M_ROOM_IN_USE" BAD_PAGINATION = "M_BAD_PAGINATION" + BAD_STATE = "M_BAD_STATE" UNKNOWN = "M_UNKNOWN" NOT_FOUND = "M_NOT_FOUND" MISSING_TOKEN = "M_MISSING_TOKEN" diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2f6359c768..26402ea9cd 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1693,7 +1693,7 @@ class FederationHandler(BaseHandler): self.auth.check(event, context.current_state) yield self._validate_keyserver(event, auth_events=context.current_state) member_handler = self.hs.get_handlers().room_member_handler - yield member_handler.change_membership(event, context) + yield member_handler.send_membership_event(event, context) else: destinations = set([x.split(":", 1)[-1] for x in (sender, room_id)]) yield self.replication_layer.forward_third_party_invite( @@ -1722,7 +1722,7 @@ class FederationHandler(BaseHandler): # TODO: Make sure the signatures actually are correct. event.signatures.update(returned_invite.signatures) member_handler = self.hs.get_handlers().room_member_handler - yield member_handler.change_membership(event, context) + yield member_handler.send_membership_event(event, context) @defer.inlineCallbacks def add_display_name_to_third_party_invite(self, event_dict, event, context): diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5805190ce8..4c7bf2bef3 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -174,30 +174,25 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def create_and_send_event(self, event_dict, ratelimit=True, - token_id=None, txn_id=None, is_guest=False): - """ Given a dict from a client, create and handle a new event. + def create_event(self, event_dict, token_id=None, txn_id=None): + """ + Given a dict from a client, create a new event. Creates an FrozenEvent object, filling out auth_events, prev_events, etc. Adds display names to Join membership events. - Persists and notifies local clients and federation. - Args: event_dict (dict): An entire event + + Returns: + Tuple of created event (FrozenEvent), Context """ builder = self.event_builder_factory.new(event_dict) self.validator.validate_new(builder) - if ratelimit: - self.ratelimit(builder.user_id) - # TODO(paul): Why does 'event' not have a 'user' object? - user = UserID.from_string(builder.user_id) - assert self.hs.is_mine(user), "User must be our own: %s" % (user,) - if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) if membership == Membership.JOIN: @@ -216,6 +211,25 @@ class MessageHandler(BaseHandler): event, context = yield self._create_new_client_event( builder=builder, ) + defer.returnValue((event, context)) + + @defer.inlineCallbacks + def send_event(self, event, context, ratelimit=True, is_guest=False): + """ + Persists and notifies local clients and federation of an event. + + Args: + event (FrozenEvent) the event to send. + context (Context) the context of the event. + ratelimit (bool): Whether to rate limit this send. + is_guest (bool): Whether the sender is a guest. + """ + user = UserID.from_string(event.sender) + + assert self.hs.is_mine(user), "User must be our own: %s" % (user,) + + if ratelimit: + self.ratelimit(event.sender) if event.is_state(): prev_state = context.current_state.get((event.type, event.state_key)) @@ -229,7 +243,7 @@ class MessageHandler(BaseHandler): if event.type == EventTypes.Member: member_handler = self.hs.get_handlers().room_member_handler - yield member_handler.change_membership(event, context, is_guest=is_guest) + yield member_handler.send_membership_event(event, context, is_guest=is_guest) else: yield self.handle_new_client_event( event=event, @@ -241,6 +255,25 @@ class MessageHandler(BaseHandler): with PreserveLoggingContext(): presence.bump_presence_active_time(user) + @defer.inlineCallbacks + def create_and_send_event(self, event_dict, ratelimit=True, + token_id=None, txn_id=None, is_guest=False): + """ + Creates an event, then sends it. + + See self.create_event and self.send_event. + """ + event, context = yield self.create_event( + event_dict, + token_id=token_id, + txn_id=txn_id + ) + yield self.send_event( + event, + context, + ratelimit=ratelimit, + is_guest=is_guest + ) defer.returnValue(event) @defer.inlineCallbacks diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a410e4394c..a1baf9d200 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -22,7 +22,7 @@ from synapse.types import UserID, RoomAlias, RoomID from synapse.api.constants import ( EventTypes, Membership, JoinRules, RoomCreationPreset, ) -from synapse.api.errors import AuthError, StoreError, SynapseError +from synapse.api.errors import AuthError, StoreError, SynapseError, Codes from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor @@ -397,7 +397,58 @@ class RoomMemberHandler(BaseHandler): remotedomains.add(member.domain) @defer.inlineCallbacks - def change_membership(self, event, context, is_guest=False): + def update_membership(self, requester, target, room_id, action, txn_id=None): + effective_membership_state = action + if action in ["kick", "unban"]: + effective_membership_state = "leave" + elif action == "forget": + effective_membership_state = "leave" + + msg_handler = self.hs.get_handlers().message_handler + + content = {"membership": unicode(effective_membership_state)} + if requester.is_guest: + content["kind"] = "guest" + + event, context = yield msg_handler.create_event( + { + "type": EventTypes.Member, + "content": content, + "room_id": room_id, + "sender": requester.user.to_string(), + "state_key": target.to_string(), + }, + token_id=requester.access_token_id, + txn_id=txn_id, + ) + + old_state = context.current_state.get((EventTypes.Member, event.state_key)) + old_membership = old_state.content.get("membership") if old_state else None + if action == "unban" and old_membership != "ban": + raise SynapseError( + 403, + "Cannot unban user who was not banned (membership=%s)" % old_membership, + errcode=Codes.BAD_STATE + ) + if old_membership == "ban" and action != "unban": + raise SynapseError( + 403, + "Cannot %s user who was is banned" % (action,), + errcode=Codes.BAD_STATE + ) + + yield msg_handler.send_event( + event, + context, + ratelimit=True, + is_guest=requester.is_guest + ) + + if action == "forget": + yield self.forget(requester.user, room_id) + + @defer.inlineCallbacks + def send_membership_event(self, event, context, is_guest=False): """ Change the membership status of a user in a room. Args: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 8b1b2b852d..85b9f253e3 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -442,7 +442,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): def register(self, http_server): # /rooms/$roomid/[invite|join|leave] PATTERNS = ("/rooms/(?P[^/]*)/" - "(?Pjoin|invite|leave|ban|kick|forget)") + "(?Pjoin|invite|leave|ban|unban|kick|forget)") register_txn_path(self, PATTERNS, http_server) @defer.inlineCallbacks @@ -451,9 +451,6 @@ class RoomMembershipRestServlet(ClientV1RestServlet): request, allow_guest=True, ) - user = requester.user - - effective_membership_action = membership_action if requester.is_guest and membership_action not in { Membership.JOIN, @@ -463,13 +460,10 @@ class RoomMembershipRestServlet(ClientV1RestServlet): content = _parse_json(request) - # target user is you unless it is an invite - state_key = user.to_string() - if membership_action == "invite" and self._has_3pid_invite_keys(content): yield self.handlers.room_member_handler.do_3pid_invite( room_id, - user, + requester.user, content["medium"], content["address"], content["id_server"], @@ -478,42 +472,21 @@ class RoomMembershipRestServlet(ClientV1RestServlet): ) defer.returnValue((200, {})) return - elif membership_action in ["invite", "ban", "kick"]: - if "user_id" in content: - state_key = content["user_id"] - else: - raise SynapseError(400, "Missing user_id key.") - - # make sure it looks like a user ID; it'll throw if it's invalid. - UserID.from_string(state_key) - if membership_action == "kick": - effective_membership_action = "leave" - elif membership_action == "forget": - effective_membership_action = "leave" - - msg_handler = self.handlers.message_handler - - content = {"membership": unicode(effective_membership_action)} - if requester.is_guest: - content["kind"] = "guest" + target = requester.user + if membership_action in ["invite", "ban", "unban", "kick"]: + if "user_id" not in content: + raise SynapseError(400, "Missing user_id key.") + target = UserID.from_string(content["user_id"]) - yield msg_handler.create_and_send_event( - { - "type": EventTypes.Member, - "content": content, - "room_id": room_id, - "sender": user.to_string(), - "state_key": state_key, - }, - token_id=requester.access_token_id, + yield self.handlers.room_member_handler.update_membership( + requester=requester, + target=target, + room_id=room_id, + action=membership_action, txn_id=txn_id, - is_guest=requester.is_guest, ) - if membership_action == "forget": - yield self.handlers.room_member_handler.forget(user, room_id) - defer.returnValue((200, {})) def _has_3pid_invite_keys(self, content): diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 97491848a3..e7a12a2ba2 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -156,7 +156,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): builder ) - yield room_handler.change_membership(event, context) + yield room_handler.send_membership_event(event, context) self.state_handler.compute_event_context.assert_called_once_with( builder @@ -232,7 +232,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ) # Actual invocation - yield room_handler.change_membership(event, context) + yield room_handler.send_membership_event(event, context) self.federation.handle_new_event.assert_called_once_with( event, destinations=set() @@ -312,7 +312,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.distributor.observe("user_left_room", leave_signal_observer) # Actual invocation - yield room_handler.change_membership(event, context) + yield room_handler.send_membership_event(event, context) self.federation.handle_new_event.assert_called_once_with( event, destinations=set(['red']) -- cgit 1.5.1 From 63485b30298f906efbdbe39311dd7503101e7602 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Jan 2016 14:11:43 +0000 Subject: Re-enable urnead notifications --- synapse/handlers/federation.py | 13 ++++++------- synapse/handlers/sync.py | 3 --- synapse/push/action_generator.py | 3 --- 3 files changed, 6 insertions(+), 13 deletions(-) (limited to 'synapse/handlers/federation.py') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 26402ea9cd..4b94940e99 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -36,7 +36,7 @@ from synapse.events.utils import prune_event from synapse.util.retryutils import NotRetryingDestination -# from synapse.push.action_generator import ActionGenerator +from synapse.push.action_generator import ActionGenerator from twisted.internet import defer @@ -244,12 +244,11 @@ class FederationHandler(BaseHandler): user = UserID.from_string(event.state_key) yield user_joined_room(self.distributor, user, event.room_id) - # Temporarily disable notifications due to performance concerns. - # if not backfilled and not event.internal_metadata.is_outlier(): - # action_generator = ActionGenerator(self.store) - # yield action_generator.handle_push_actions_for_event( - # event, self - # ) + if not backfilled and not event.internal_metadata.is_outlier(): + action_generator = ActionGenerator(self.store) + yield action_generator.handle_push_actions_for_event( + event, self + ) @defer.inlineCallbacks def _filter_events_for_server(self, server_name, room_id, events): diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1942268c3c..52202d8e63 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -841,9 +841,6 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def unread_notifs_for_room_id(self, room_id, sync_config, ephemeral_by_room): - # Temporarily disable notifications due to performance concerns. - defer.returnValue([]) - last_unread_event_id = self.last_read_event_id_for_room_and_user( room_id, sync_config.user.to_string(), ephemeral_by_room ) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 73467f3adc..4cf94f6c61 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -36,9 +36,6 @@ class ActionGenerator: @defer.inlineCallbacks def handle_push_actions_for_event(self, event, handler): - # Temporarily disable notifications due to performance concerns. - return - if event.type == EventTypes.Redaction and event.redacts is not None: yield self.store.remove_push_actions_for_event_id( event.room_id, event.redacts -- cgit 1.5.1 From 5a7d1ecffcab7a94caf70471a2eec56eb868573c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Jan 2016 16:01:05 +0000 Subject: Add regex cache. Only caculate push actions for users that have sent read receipts, and are on that server --- synapse/handlers/_base.py | 2 +- synapse/handlers/federation.py | 2 +- synapse/push/action_generator.py | 7 ++++--- synapse/push/bulk_push_rule_evaluator.py | 15 ++++++++++----- synapse/push/push_rule_evaluator.py | 20 +++++++++++++++++--- synapse/server.py | 4 ++++ synapse/storage/receipts.py | 14 +++++++++++++- 7 files changed, 50 insertions(+), 14 deletions(-) (limited to 'synapse/handlers/federation.py') diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 2d1167296a..5c7617de44 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -266,7 +266,7 @@ class BaseHandler(object): event, context=context ) - action_generator = ActionGenerator(self.store) + action_generator = ActionGenerator(self.hs) yield action_generator.handle_push_actions_for_event( event, self ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4b94940e99..6c19d6ae8c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -245,7 +245,7 @@ class FederationHandler(BaseHandler): yield user_joined_room(self.distributor, user, event.room_id) if not backfilled and not event.internal_metadata.is_outlier(): - action_generator = ActionGenerator(self.store) + action_generator = ActionGenerator(self.hs) yield action_generator.handle_push_actions_for_event( event, self ) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index 4cf94f6c61..1d2e558f9a 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -25,8 +25,9 @@ logger = logging.getLogger(__name__) class ActionGenerator: - def __init__(self, store): - self.store = store + def __init__(self, hs): + self.hs = hs + self.store = hs.get_datastore() # really we want to get all user ids and all profile tags too, # since we want the actions for each profile tag for every user and # also actions for a client with no profile tag for each user. @@ -42,7 +43,7 @@ class ActionGenerator: ) bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id( - event.room_id, self.store + event.room_id, self.hs, self.store ) actions_by_user = yield bulk_evaluator.action_for_event_by_user(event, handler) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index efd686fa6e..1000ae6301 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -36,6 +36,7 @@ def decode_rule_json(rule): @defer.inlineCallbacks def _get_rules(room_id, user_ids, store): rules_by_user = yield store.bulk_get_push_rules(user_ids) + rules_by_user = { uid: baserules.list_with_base_rules([ decode_rule_json(rule_list) @@ -47,12 +48,16 @@ def _get_rules(room_id, user_ids, store): @defer.inlineCallbacks -def evaluator_for_room_id(room_id, store): - users = yield store.get_users_in_room(room_id) - rules_by_user = yield _get_rules(room_id, users, store) +def evaluator_for_room_id(room_id, hs, store): + results = yield store.get_receipts_for_room(room_id, "m.read") + user_ids = [ + row["user_id"] for row in results + if hs.is_mine_id(row["user_id"]) + ] + rules_by_user = yield _get_rules(room_id, user_ids, store) defer.returnValue(BulkPushRuleEvaluator( - room_id, rules_by_user, users, store + room_id, rules_by_user, user_ids, store )) @@ -129,7 +134,7 @@ def _condition_checker(evaluator, conditions, uid, display_name, cache): res = evaluator.matches(cond, uid, display_name, None) if _id: - cache[_id] = res + cache[_id] = bool(res) if not res: return False diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 4654994d2d..753b6469e2 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -22,6 +22,7 @@ import simplejson as json import re from synapse.types import UserID +from synapse.util.caches.lrucache import LruCache logger = logging.getLogger(__name__) @@ -277,18 +278,18 @@ def _glob_matches(glob, value, word_boundary=False): ) if word_boundary: r = r"\b%s\b" % (r,) - r = re.compile(r, flags=re.IGNORECASE) + r = _compile_regex(r) return r.search(value) else: r = r + "$" - r = re.compile(r, flags=re.IGNORECASE) + r = _compile_regex(r) return r.match(value) elif word_boundary: r = re.escape(glob) r = r"\b%s\b" % (r,) - r = re.compile(r, flags=re.IGNORECASE) + r = _compile_regex(r) return r.search(value) else: @@ -306,3 +307,16 @@ def _flatten_dict(d, prefix=[], result={}): _flatten_dict(value, prefix=(prefix+[key]), result=result) return result + + +regex_cache = LruCache(100000) + + +def _compile_regex(regex_str): + r = regex_cache.get(regex_str, None) + if r: + return r + + r = re.compile(regex_str, flags=re.IGNORECASE) + regex_cache[regex_str] = r + return r diff --git a/synapse/server.py b/synapse/server.py index ffd4f936d0..63f9059837 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -96,6 +96,7 @@ class BaseHomeServer(object): hostname : The hostname for the server. """ self.hostname = hostname + self.hostname_with_colon = ":" + hostname self._building = {} # Other kwargs are explicit dependencies @@ -139,6 +140,9 @@ class BaseHomeServer(object): def is_mine(self, domain_specific_string): return domain_specific_string.domain == self.hostname + def is_mine_id(self, string): + return string.endswith(self.hostname_with_colon) + # Build magic accessors for every dependency for depname in BaseHomeServer.DEPENDENCIES: BaseHomeServer._make_dependency_method(depname) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 21cf88b3da..c80e576620 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -14,7 +14,7 @@ # limitations under the License. from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList +from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList, cached from synapse.util.caches import cache_counter, caches_by_name from twisted.internet import defer @@ -33,6 +33,18 @@ class ReceiptsStore(SQLBaseStore): self._receipts_stream_cache = _RoomStreamChangeCache() + @cached(num_args=2) + def get_receipts_for_room(self, room_id, receipt_type): + return self._simple_select_list( + table="receipts_linearized", + keyvalues={ + "room_id": room_id, + "receipt_type": receipt_type, + }, + retcols=("user_id", "event_id"), + desc="get_receipts_for_room", + ) + @defer.inlineCallbacks def get_linearized_receipts_for_rooms(self, room_ids, to_key, from_key=None): """Get receipts for multiple rooms for sending to clients. -- cgit 1.5.1