From e3e2fc3255665ac888f3e0c0c6e2be39d5fda5f3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 15 Jan 2015 16:17:21 +0000 Subject: Don't make the pushers' event streams cause people to appear online --- synapse/handlers/events.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index c9ade253dd..54ab27004f 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -47,11 +47,11 @@ class EventStreamHandler(BaseHandler): @defer.inlineCallbacks @log_function def get_stream(self, auth_user_id, pagin_config, timeout=0, - as_client_event=True): + as_client_event=True, affect_presence=True): auth_user = self.hs.parse_userid(auth_user_id) try: - if auth_user not in self._streams_per_user: + if affect_presence and auth_user not in self._streams_per_user: self._streams_per_user[auth_user] = 0 if auth_user in self._stop_timer_per_user: try: @@ -64,7 +64,7 @@ class EventStreamHandler(BaseHandler): yield self.distributor.fire( "started_user_eventstream", auth_user ) - self._streams_per_user[auth_user] += 1 + self._streams_per_user[auth_user] += 1 if pagin_config.from_token is None: pagin_config.from_token = None @@ -92,27 +92,28 @@ class EventStreamHandler(BaseHandler): defer.returnValue(chunk) finally: - self._streams_per_user[auth_user] -= 1 - if not self._streams_per_user[auth_user]: - del self._streams_per_user[auth_user] - - # 10 seconds of grace to allow the client to reconnect again - # before we think they're gone - def _later(): - logger.debug( - "_later stopped_user_eventstream %s", auth_user - ) + if affect_presence: + self._streams_per_user[auth_user] -= 1 + if not self._streams_per_user[auth_user]: + del self._streams_per_user[auth_user] + + # 10 seconds of grace to allow the client to reconnect again + # before we think they're gone + def _later(): + logger.debug( + "_later stopped_user_eventstream %s", auth_user + ) - self._stop_timer_per_user.pop(auth_user, None) + self._stop_timer_per_user.pop(auth_user, None) - yield self.distributor.fire( - "stopped_user_eventstream", auth_user - ) + yield self.distributor.fire( + "stopped_user_eventstream", auth_user + ) - logger.debug("Scheduling _later: for %s", auth_user) - self._stop_timer_per_user[auth_user] = ( - self.clock.call_later(30, _later) - ) + logger.debug("Scheduling _later: for %s", auth_user) + self._stop_timer_per_user[auth_user] = ( + self.clock.call_later(30, _later) + ) class EventHandler(BaseHandler): -- cgit 1.4.1 From 5759bec43cb52862a8d455afb8cd9d1c5660bc3d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 11:47:15 +0000 Subject: Replace hs.parse_userid with UserID.from_string --- synapse/api/auth.py | 9 +++++---- synapse/handlers/_base.py | 5 +++-- synapse/handlers/events.py | 3 ++- synapse/handlers/federation.py | 13 +++++++------ synapse/handlers/message.py | 13 +++++++------ synapse/handlers/presence.py | 23 ++++++++++++----------- synapse/handlers/profile.py | 3 ++- synapse/handlers/room.py | 14 +++++++------- synapse/handlers/typing.py | 3 ++- synapse/rest/client/v1/admin.py | 4 +++- synapse/rest/client/v1/presence.py | 15 ++++++++------- synapse/rest/client/v1/profile.py | 13 +++++++------ synapse/rest/client/v1/room.py | 5 +++-- synapse/server.py | 6 ------ synapse/storage/roommember.py | 5 +++-- tests/handlers/test_presence.py | 29 +++++++++++++++-------------- tests/handlers/test_presencelike.py | 9 +++++---- tests/handlers/test_profile.py | 8 ++++---- tests/handlers/test_room.py | 9 +++++---- tests/handlers/test_typing.py | 7 ++++--- tests/rest/client/v1/test_presence.py | 19 ++++++++++--------- tests/rest/client/v1/test_profile.py | 3 ++- tests/rest/client/v1/test_rooms.py | 21 +++++++++------------ tests/rest/client/v1/test_typing.py | 5 +++-- tests/storage/test_presence.py | 5 +++-- tests/storage/test_profile.py | 3 ++- tests/storage/test_redaction.py | 5 +++-- tests/storage/test_room.py | 3 ++- tests/storage/test_roommember.py | 7 ++++--- tests/storage/test_stream.py | 5 +++-- tests/test_types.py | 6 ------ 31 files changed, 145 insertions(+), 133 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e31482cfaa..a342a0e0da 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,6 +21,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor +from synapse.types import UserID import logging @@ -104,7 +105,7 @@ class Auth(object): for event in curr_state: if event.type == EventTypes.Member: try: - if self.hs.parse_userid(event.state_key).domain != host: + if UserID.from_string(event.state_key).domain != host: continue except: logger.warn("state_key not user_id: %s", event.state_key) @@ -337,7 +338,7 @@ class Auth(object): user_info = { "admin": bool(ret.get("admin", False)), "device_id": ret.get("device_id"), - "user": self.hs.parse_userid(ret.get("name")), + "user": UserID.from_string(ret.get("name")), } defer.returnValue(user_info) @@ -461,7 +462,7 @@ class Auth(object): "You are not allowed to set others state" ) else: - sender_domain = self.hs.parse_userid( + sender_domain = UserID.from_string( event.user_id ).domain @@ -496,7 +497,7 @@ class Auth(object): # Validate users for k, v in user_list.items(): try: - self.hs.parse_userid(k) + UserID.from_string(k) except: raise SynapseError(400, "Not a valid user_id: %s" % (k,)) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index f33d17a31e..1773fa20aa 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -19,6 +19,7 @@ from synapse.api.errors import LimitExceededError, SynapseError from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes +from synapse.types import UserID import logging @@ -113,7 +114,7 @@ class BaseHandler(object): if event.type == EventTypes.Member: if event.content["membership"] == Membership.INVITE: - invitee = self.hs.parse_userid(event.state_key) + invitee = UserID.from_string(event.state_key) if not self.hs.is_mine(invitee): # TODO: Can we add signature from remote server in a nicer # way? If we have been invited by a remote server, we need @@ -134,7 +135,7 @@ class BaseHandler(object): if k[0] == EventTypes.Member: if s.content["membership"] == Membership.JOIN: destinations.add( - self.hs.parse_userid(s.state_key).domain + UserID.from_string(s.state_key).domain ) except SynapseError: logger.warn( diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 103bc67c42..01e67b0818 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -17,6 +17,7 @@ from twisted.internet import defer from synapse.util.logcontext import PreserveLoggingContext from synapse.util.logutils import log_function +from synapse.types import UserID from ._base import BaseHandler @@ -48,7 +49,7 @@ class EventStreamHandler(BaseHandler): @log_function def get_stream(self, auth_user_id, pagin_config, timeout=0, as_client_event=True): - auth_user = self.hs.parse_userid(auth_user_id) + auth_user = UserID.from_string(auth_user_id) try: if auth_user not in self._streams_per_user: diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 81203bf1a3..bcdcc90a18 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -28,6 +28,7 @@ from synapse.crypto.event_signing import ( compute_event_signature, check_event_content_hash, add_hashes_and_signatures, ) +from synapse.types import UserID from syutil.jsonutil import encode_canonical_json from twisted.internet import defer @@ -227,7 +228,7 @@ class FederationHandler(BaseHandler): extra_users = [] if event.type == EventTypes.Member: target_user_id = event.state_key - target_user = self.hs.parse_userid(target_user_id) + target_user = UserID.from_string(target_user_id) extra_users.append(target_user) yield self.notifier.on_new_room_event( @@ -236,7 +237,7 @@ class FederationHandler(BaseHandler): if event.type == EventTypes.Member: if event.membership == Membership.JOIN: - user = self.hs.parse_userid(event.state_key) + user = UserID.from_string(event.state_key) yield self.distributor.fire( "user_joined_room", user=user, room_id=event.room_id ) @@ -491,7 +492,7 @@ class FederationHandler(BaseHandler): extra_users = [] if event.type == EventTypes.Member: target_user_id = event.state_key - target_user = self.hs.parse_userid(target_user_id) + target_user = UserID.from_string(target_user_id) extra_users.append(target_user) yield self.notifier.on_new_room_event( @@ -500,7 +501,7 @@ class FederationHandler(BaseHandler): if event.type == EventTypes.Member: if event.content["membership"] == Membership.JOIN: - user = self.hs.parse_userid(event.state_key) + user = UserID.from_string(event.state_key) yield self.distributor.fire( "user_joined_room", user=user, room_id=event.room_id ) @@ -514,7 +515,7 @@ class FederationHandler(BaseHandler): if k[0] == EventTypes.Member: if s.content["membership"] == Membership.JOIN: destinations.add( - self.hs.parse_userid(s.state_key).domain + UserID.from_string(s.state_key).domain ) except: logger.warn( @@ -565,7 +566,7 @@ class FederationHandler(BaseHandler): backfilled=False, ) - target_user = self.hs.parse_userid(event.state_key) + target_user = UserID.from_string(event.state_key) yield self.notifier.on_new_room_event( event, extra_users=[target_user], ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index f2a2f16933..6a1104a890 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -20,6 +20,7 @@ from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig from synapse.events.validator import EventValidator from synapse.util.logcontext import PreserveLoggingContext +from synapse.types import UserID from ._base import BaseHandler @@ -89,7 +90,7 @@ class MessageHandler(BaseHandler): yield self.hs.get_event_sources().get_current_token() ) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) events, next_key = yield data_source.get_pagination_rows( user, pagin_config.get_source_config("room"), room_id @@ -130,13 +131,13 @@ class MessageHandler(BaseHandler): if ratelimit: self.ratelimit(builder.user_id) # TODO(paul): Why does 'event' not have a 'user' object? - user = self.hs.parse_userid(builder.user_id) + user = UserID.from_string(builder.user_id) assert self.hs.is_mine(user), "User must be our own: %s" % (user,) if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) if membership == Membership.JOIN: - joinee = self.hs.parse_userid(builder.state_key) + joinee = UserID.from_string(builder.state_key) # If event doesn't include a display name, add one. yield self.distributor.fire( "collect_presencelike_data", @@ -237,7 +238,7 @@ class MessageHandler(BaseHandler): membership_list=[Membership.INVITE, Membership.JOIN] ) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) rooms_ret = [] @@ -316,7 +317,7 @@ class MessageHandler(BaseHandler): # TODO(paul): I wish I was called with user objects not user_id # strings... - auth_user = self.hs.parse_userid(user_id) + auth_user = UserID.from_string(user_id) # TODO: These concurrently state_tuples = yield self.state_handler.get_current_state(room_id) @@ -349,7 +350,7 @@ class MessageHandler(BaseHandler): for m in room_members: try: member_presence = yield presence_handler.get_state( - target_user=self.hs.parse_userid(m.user_id), + target_user=UserID.from_string(m.user_id), auth_user=auth_user, as_event=True, ) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 8aeed99274..d66bfea7b1 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -20,6 +20,7 @@ from synapse.api.constants import PresenceState from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext +from synapse.types import UserID from ._base import BaseHandler @@ -96,22 +97,22 @@ class PresenceHandler(BaseHandler): self.federation.register_edu_handler( "m.presence_invite", lambda origin, content: self.invite_presence( - observed_user=hs.parse_userid(content["observed_user"]), - observer_user=hs.parse_userid(content["observer_user"]), + observed_user=UserID.from_string(content["observed_user"]), + observer_user=UserID.from_string(content["observer_user"]), ) ) self.federation.register_edu_handler( "m.presence_accept", lambda origin, content: self.accept_presence( - observed_user=hs.parse_userid(content["observed_user"]), - observer_user=hs.parse_userid(content["observer_user"]), + observed_user=UserID.from_string(content["observed_user"]), + observer_user=UserID.from_string(content["observer_user"]), ) ) self.federation.register_edu_handler( "m.presence_deny", lambda origin, content: self.deny_presence( - observed_user=hs.parse_userid(content["observed_user"]), - observer_user=hs.parse_userid(content["observer_user"]), + observed_user=UserID.from_string(content["observed_user"]), + observer_user=UserID.from_string(content["observer_user"]), ) ) @@ -418,7 +419,7 @@ class PresenceHandler(BaseHandler): ) for p in presence: - observed_user = self.hs.parse_userid(p.pop("observed_user_id")) + observed_user = UserID.from_string(p.pop("observed_user_id")) p["observed_user"] = observed_user p.update(self._get_or_offline_usercache(observed_user).get_state()) if "last_active" in p: @@ -441,7 +442,7 @@ class PresenceHandler(BaseHandler): user.localpart, accepted=True ) target_users = set([ - self.hs.parse_userid(x["observed_user_id"]) for x in presence + UserID.from_string(x["observed_user_id"]) for x in presence ]) # Also include people in all my rooms @@ -646,7 +647,7 @@ class PresenceHandler(BaseHandler): deferreds = [] for push in content.get("push", []): - user = self.hs.parse_userid(push["user_id"]) + user = UserID.from_string(push["user_id"]) logger.debug("Incoming presence update from %s", user) @@ -694,7 +695,7 @@ class PresenceHandler(BaseHandler): del self._user_cachemap[user] for poll in content.get("poll", []): - user = self.hs.parse_userid(poll) + user = UserID.from_string(poll) if not self.hs.is_mine(user): continue @@ -709,7 +710,7 @@ class PresenceHandler(BaseHandler): deferreds.append(self._push_presence_remote(user, origin)) for unpoll in content.get("unpoll", []): - user = self.hs.parse_userid(unpoll) + user = UserID.from_string(unpoll) if not self.hs.is_mine(user): continue diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 7777d3cc94..03b2159c53 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException from synapse.api.constants import EventTypes, Membership from synapse.util.logcontext import PreserveLoggingContext +from synapse.types import UserID from ._base import BaseHandler @@ -169,7 +170,7 @@ class ProfileHandler(BaseHandler): @defer.inlineCallbacks def on_profile_query(self, args): - user = self.hs.parse_userid(args["user_id"]) + user = UserID.from_string(args["user_id"]) if not self.hs.is_mine(user): raise SynapseError(400, "User is not hosted on this Home Server") diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6d0db18e51..0242288c4e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -64,7 +64,7 @@ class RoomCreationHandler(BaseHandler): invite_list = config.get("invite", []) for i in invite_list: try: - self.hs.parse_userid(i) + UserID.from_string(i) except: raise SynapseError(400, "Invalid user_id: %s" % (i,)) @@ -114,7 +114,7 @@ class RoomCreationHandler(BaseHandler): servers=[self.hs.hostname], ) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) creation_events = self._create_events_for_new_room( user, room_id, is_public=is_public ) @@ -250,7 +250,7 @@ class RoomMemberHandler(BaseHandler): users = yield self.store.get_users_in_room(room_id) - defer.returnValue([hs.parse_userid(u) for u in users]) + defer.returnValue([UserID.from_string(u) for u in users]) @defer.inlineCallbacks def fetch_room_distributions_into(self, room_id, localusers=None, @@ -368,7 +368,7 @@ class RoomMemberHandler(BaseHandler): ) if prev_state and prev_state.membership == Membership.JOIN: - user = self.hs.parse_userid(event.user_id) + user = UserID.from_string(event.user_id) self.distributor.fire( "user_left_room", user=user, room_id=event.room_id ) @@ -412,7 +412,7 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def _do_join(self, event, context, room_host=None, do_auth=True): - joinee = self.hs.parse_userid(event.state_key) + joinee = UserID.from_string(event.state_key) # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id @@ -476,7 +476,7 @@ class RoomMemberHandler(BaseHandler): do_auth=do_auth, ) - user = self.hs.parse_userid(event.user_id) + user = UserID.from_string(event.user_id) yield self.distributor.fire( "user_joined_room", user=user, room_id=room_id ) @@ -526,7 +526,7 @@ class RoomMemberHandler(BaseHandler): do_auth): yield run_on_reactor() - target_user = self.hs.parse_userid(event.state_key) + target_user = UserID.from_string(event.state_key) yield self.handle_new_client_event( event, diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index cd9638dd04..c69787005f 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError, AuthError +from synapse.types import UserID import logging @@ -185,7 +186,7 @@ class TypingNotificationHandler(BaseHandler): @defer.inlineCallbacks def _recv_edu(self, origin, content): room_id = content["room_id"] - user = self.homeserver.parse_userid(content["user_id"]) + user = UserID.from_string(content["user_id"]) localusers = set() diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 0aa83514c8..4aefb94053 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -16,6 +16,8 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError +from synapse.types import UserID + from base import RestServlet, client_path_pattern import logging @@ -28,7 +30,7 @@ class WhoisRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - target_user = self.hs.parse_userid(user_id) + target_user = UserID.from_string(user_id) auth_user = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index ca4d2d21f0..22fcb7d7d0 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -18,7 +18,8 @@ from twisted.internet import defer from synapse.api.errors import SynapseError -from base import RestServlet, client_path_pattern +from synapse.types import UserID +from .base import RestServlet, client_path_pattern import json import logging @@ -32,7 +33,7 @@ class PresenceStatusRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( target_user=user, auth_user=auth_user) @@ -42,7 +43,7 @@ class PresenceStatusRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) state = {} try: @@ -77,7 +78,7 @@ class PresenceListRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) if not self.hs.is_mine(user): raise SynapseError(400, "User not hosted on this Home Server") @@ -97,7 +98,7 @@ class PresenceListRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) if not self.hs.is_mine(user): raise SynapseError(400, "User not hosted on this Home Server") @@ -118,7 +119,7 @@ class PresenceListRestServlet(RestServlet): raise SynapseError(400, "Bad invite value.") if len(u) == 0: continue - invited_user = self.hs.parse_userid(u) + invited_user = UserID.from_string(u) yield self.handlers.presence_handler.send_invite( observer_user=user, observed_user=invited_user ) @@ -129,7 +130,7 @@ class PresenceListRestServlet(RestServlet): raise SynapseError(400, "Bad drop value.") if len(u) == 0: continue - dropped_user = self.hs.parse_userid(u) + dropped_user = UserID.from_string(u) yield self.handlers.presence_handler.drop( observer_user=user, observed_user=dropped_user ) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index dc6eb424b0..39297930c8 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -16,7 +16,8 @@ """ This module contains REST servlets to do with profile: /profile/ """ from twisted.internet import defer -from base import RestServlet, client_path_pattern +from .base import RestServlet, client_path_pattern +from synapse.types import UserID import json @@ -26,7 +27,7 @@ class ProfileDisplaynameRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) displayname = yield self.handlers.profile_handler.get_displayname( user, @@ -37,7 +38,7 @@ class ProfileDisplaynameRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) try: content = json.loads(request.content.read()) @@ -59,7 +60,7 @@ class ProfileAvatarURLRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) avatar_url = yield self.handlers.profile_handler.get_avatar_url( user, @@ -70,7 +71,7 @@ class ProfileAvatarURLRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) try: content = json.loads(request.content.read()) @@ -92,7 +93,7 @@ class ProfileRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) displayname = yield self.handlers.profile_handler.get_displayname( user, diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 48bba2a5f3..c5837b3403 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -20,6 +20,7 @@ from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID import json import logging @@ -289,7 +290,7 @@ class RoomMemberListRestServlet(RestServlet): for event in members["chunk"]: # FIXME: should probably be state_key here, not user_id - target_user = self.hs.parse_userid(event["user_id"]) + target_user = UserID.from_string(event["user_id"]) # Presence is an optional cache; don't fail if we can't fetch it try: presence_handler = self.handlers.presence_handler @@ -478,7 +479,7 @@ class RoomTypingRestServlet(RestServlet): auth_user = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) - target_user = self.hs.parse_userid(urllib.unquote(user_id)) + target_user = UserID.from_string(urllib.unquote(user_id)) content = _parse_json(request) diff --git a/synapse/server.py b/synapse/server.py index 476d809374..52a21aaf78 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -127,12 +127,6 @@ class BaseHomeServer(object): # TODO: Why are these parse_ methods so high up along with other globals? # Surely these should be in a util package or in the api package? - # Other utility methods - def parse_userid(self, s): - """Parse the string given by 's' as a User ID and return a UserID - object.""" - return UserID.from_string(s) - def parse_roomalias(self, s): """Parse the string given by 's' as a Room Alias and return a RoomAlias object.""" diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index e59e65529b..c69dd995ce 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -20,6 +20,7 @@ from collections import namedtuple from ._base import SQLBaseStore from synapse.api.constants import Membership +from synapse.types import UserID import logging @@ -39,7 +40,7 @@ class RoomMemberStore(SQLBaseStore): """ try: target_user_id = event.state_key - domain = self.hs.parse_userid(target_user_id).domain + domain = UserID.from_string(target_user_id).domain except: logger.exception( "Failed to parse target_user_id=%s", target_user_id @@ -84,7 +85,7 @@ class RoomMemberStore(SQLBaseStore): for e in member_events: try: joined_domains.add( - self.hs.parse_userid(e.state_key).domain + UserID.from_string(e.state_key).domain ) except: # FIXME: How do we deal with invalid user ids in the db? diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 56e90177f1..5621a8afaf 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -17,7 +17,7 @@ from tests import unittest from twisted.internet import defer, reactor -from mock import Mock, call, ANY, NonCallableMock, patch +from mock import Mock, call, ANY, NonCallableMock import json from tests.utils import ( @@ -31,6 +31,7 @@ from synapse.api.errors import SynapseError from synapse.handlers.presence import PresenceHandler, UserPresenceCache from synapse.streams.config import SourcePaginationConfig from synapse.storage.transactions import DestinationsTable +from synapse.types import UserID OFFLINE = PresenceState.OFFLINE UNAVAILABLE = PresenceState.UNAVAILABLE @@ -170,9 +171,9 @@ class PresenceTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp_users(self, hs): # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") + self.u_clementine = UserID.from_string("@clementine:test") for u in self.u_apple, self.u_banana, self.u_clementine: yield self.datastore.create_presence(u.localpart) @@ -182,10 +183,10 @@ class PresenceTestCase(unittest.TestCase): ) # ID of a local user that does not exist - self.u_durian = hs.parse_userid("@durian:test") + self.u_durian = UserID.from_string("@durian:test") # A remote user - self.u_cabbage = hs.parse_userid("@cabbage:elsewhere") + self.u_cabbage = UserID.from_string("@cabbage:elsewhere") class MockedDatastorePresenceTestCase(PresenceTestCase): @@ -250,16 +251,16 @@ class MockedDatastorePresenceTestCase(PresenceTestCase): @defer.inlineCallbacks def setUp_users(self, hs): # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") - self.u_durian = hs.parse_userid("@durian:test") - self.u_elderberry = hs.parse_userid("@elderberry:test") - self.u_fig = hs.parse_userid("@fig:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") + self.u_clementine = UserID.from_string("@clementine:test") + self.u_durian = UserID.from_string("@durian:test") + self.u_elderberry = UserID.from_string("@elderberry:test") + self.u_fig = UserID.from_string("@fig:test") # Remote user - self.u_onion = hs.parse_userid("@onion:farm") - self.u_potato = hs.parse_userid("@potato:remote") + self.u_onion = UserID.from_string("@onion:farm") + self.u_potato = UserID.from_string("@potato:remote") yield diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 0584e4c8b9..3cdbb186ae 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -27,6 +27,7 @@ from synapse.server import HomeServer from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler from synapse.handlers.profile import ProfileHandler +from synapse.types import UserID OFFLINE = PresenceState.OFFLINE @@ -136,12 +137,12 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): lambda u: defer.succeed([])) # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") + self.u_clementine = UserID.from_string("@clementine:test") # Remote user - self.u_potato = hs.parse_userid("@potato:remote") + self.u_potato = UserID.from_string("@potato:remote") self.mock_get_joined = ( self.datastore.get_rooms_for_user_where_membership_is diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 25b172aa5e..7b9590c110 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -22,7 +22,7 @@ from mock import Mock, NonCallableMock from synapse.api.errors import AuthError from synapse.server import HomeServer from synapse.handlers.profile import ProfileHandler -from synapse.api.constants import Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -71,9 +71,9 @@ class ProfileTestCase(unittest.TestCase): self.store = hs.get_datastore() - self.frank = hs.parse_userid("@1234ABCD:test") - self.bob = hs.parse_userid("@4567:test") - self.alice = hs.parse_userid("@alice:remote") + self.frank = UserID.from_string("@1234ABCD:test") + self.bob = UserID.from_string("@4567:test") + self.alice = UserID.from_string("@alice:remote") yield self.store.create_profile(self.frank.localpart) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index d3253b48b8..9a23b3812d 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -15,12 +15,13 @@ from twisted.internet import defer -from tests import unittest +from .. import unittest from synapse.api.constants import EventTypes, Membership from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler from synapse.handlers.profile import ProfileHandler from synapse.server import HomeServer +from synapse.types import UserID from ..utils import MockKey from mock import Mock, NonCallableMock @@ -164,7 +165,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): event, context=context, ) self.notifier.on_new_room_event.assert_called_once_with( - event, extra_users=[self.hs.parse_userid(target_user_id)] + event, extra_users=[UserID.from_string(target_user_id)] ) self.assertFalse(self.datastore.get_room.called) self.assertFalse(self.datastore.store_room.called) @@ -174,7 +175,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def test_simple_join(self): room_id = "!foo:red" user_id = "@bob:red" - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) join_signal_observer = Mock() self.distributor.observe("user_joined_room", join_signal_observer) @@ -252,7 +253,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def test_simple_leave(self): room_id = "!foo:red" user_id = "@bob:red" - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) builder = self.hs.get_event_builder_factory().new({ "type": EventTypes.Member, diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 6a498b23a4..8a7fc028d1 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -27,6 +27,7 @@ from synapse.server import HomeServer from synapse.handlers.typing import TypingNotificationHandler from synapse.storage.transactions import DestinationsTable +from synapse.types import UserID def _expect_edu(destination, edu_type, content, origin="test"): @@ -153,11 +154,11 @@ class TypingNotificationsTestCase(unittest.TestCase): self.auth.check_joined_room = check_joined_room # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") # Remote user - self.u_onion = hs.parse_userid("@onion:farm") + self.u_onion = UserID.from_string("@onion:farm") @defer.inlineCallbacks def test_started_typing_local(self): diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 783720ac29..65d5cc4916 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -27,6 +27,7 @@ from synapse.handlers.presence import PresenceHandler from synapse.server import HomeServer from synapse.rest.client.v1 import presence from synapse.rest.client.v1 import events +from synapse.types import UserID OFFLINE = PresenceState.OFFLINE @@ -71,7 +72,7 @@ class PresenceStateTestCase(unittest.TestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(myid), + "user": UserID.from_string(myid), "admin": False, "device_id": None, } @@ -90,7 +91,7 @@ class PresenceStateTestCase(unittest.TestCase): presence.register_servlets(hs, self.mock_resource) - self.u_apple = hs.parse_userid(myid) + self.u_apple = UserID.from_string(myid) @defer.inlineCallbacks def test_get_my_status(self): @@ -161,12 +162,12 @@ class PresenceListTestCase(unittest.TestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(myid), + "user": UserID.from_string(myid), "admin": False, "device_id": None, } - room_member_handler = hs.handlers.room_member_handler = Mock( + hs.handlers.room_member_handler = Mock( spec=[ "get_rooms_for_user", ] @@ -176,8 +177,8 @@ class PresenceListTestCase(unittest.TestCase): presence.register_servlets(hs, self.mock_resource) - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") @defer.inlineCallbacks def test_get_my_list(self): @@ -281,7 +282,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.get_clock().time_msec.return_value = 1000000 def _get_user_by_req(req=None): - return hs.parse_userid(myid) + return UserID.from_string(myid) hs.get_auth().get_user_by_req = _get_user_by_req @@ -322,8 +323,8 @@ class PresenceEventStreamTestCase(unittest.TestCase): self.presence = hs.get_handlers().presence_handler - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") @defer.inlineCallbacks def test_shortpoll(self): diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 5b5c3edc22..39cd68d829 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -24,6 +24,7 @@ from ....utils import MockHttpResource, MockKey from synapse.api.errors import SynapseError, AuthError from synapse.server import HomeServer +from synapse.types import UserID from synapse.rest.client.v1 import profile @@ -57,7 +58,7 @@ class ProfileTestCase(unittest.TestCase): ) def _get_user_by_req(request=None): - return hs.parse_userid(myid) + return UserID.from_string(myid) hs.get_auth().get_user_by_req = _get_user_by_req diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 12f8040541..76ed550b75 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -22,13 +22,10 @@ import synapse.rest.client.v1.room from synapse.api.constants import Membership from synapse.server import HomeServer +from synapse.types import UserID -from tests import unittest - -# python imports import json import urllib -import types from ....utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase @@ -70,7 +67,7 @@ class RoomPermissionsTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -466,7 +463,7 @@ class RoomsMemberListTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -555,7 +552,7 @@ class RoomsCreateTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -657,7 +654,7 @@ class RoomTopicTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -773,7 +770,7 @@ class RoomMemberStateTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -909,7 +906,7 @@ class RoomMessagesTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -1013,7 +1010,7 @@ class RoomInitialSyncTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -1028,7 +1025,7 @@ class RoomInitialSyncTestCase(RestTestCase): # Since I'm getting my own presence I need to exist as far as presence # is concerned. hs.get_handlers().presence_handler.registered_user( - hs.parse_userid(self.user_id) + UserID.from_string(self.user_id) ) # create the room diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 647bcebfd8..c89b37d004 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -20,6 +20,7 @@ from twisted.internet import defer import synapse.rest.client.v1.room from synapse.server import HomeServer +from synapse.types import UserID from ....utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase @@ -69,7 +70,7 @@ class RoomTypingTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -82,7 +83,7 @@ class RoomTypingTestCase(RestTestCase): def get_room_members(room_id): if room_id == self.room_id: - return defer.succeed([hs.parse_userid(self.user_id)]) + return defer.succeed([UserID.from_string(self.user_id)]) else: return defer.succeed([]) diff --git a/tests/storage/test_presence.py b/tests/storage/test_presence.py index 9655d3cf42..1ab193736b 100644 --- a/tests/storage/test_presence.py +++ b/tests/storage/test_presence.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.presence import PresenceStore +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockClock @@ -37,8 +38,8 @@ class PresenceStoreTestCase(unittest.TestCase): self.store = PresenceStore(hs) - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") @defer.inlineCallbacks def test_state(self): diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index 5d36723c28..84381241bc 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.profile import ProfileStore +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool @@ -36,7 +37,7 @@ class ProfileStoreTestCase(unittest.TestCase): self.store = ProfileStore(hs) - self.u_frank = hs.parse_userid("@frank:test") + self.u_frank = UserID.from_string("@frank:test") @defer.inlineCallbacks def test_displayname(self): diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index 9806fbc69b..a16ccad881 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -48,8 +49,8 @@ class RedactionTestCase(unittest.TestCase): self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler - self.u_alice = hs.parse_userid("@alice:test") - self.u_bob = hs.parse_userid("@bob:test") + self.u_alice = UserID.from_string("@alice:test") + self.u_bob = UserID.from_string("@bob:test") self.room1 = hs.parse_roomid("!abc123:test") diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index e7739776ec..c6bfde069a 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool @@ -40,7 +41,7 @@ class RoomStoreTestCase(unittest.TestCase): self.room = hs.parse_roomid("!abcde:test") self.alias = hs.parse_roomalias("#a-room-name:test") - self.u_creator = hs.parse_userid("@creator:test") + self.u_creator = UserID.from_string("@creator:test") yield self.store.store_room(self.room.to_string(), room_creator_user_id=self.u_creator.to_string(), diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index a23a8189df..6b7930b1d8 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -49,11 +50,11 @@ class RoomMemberStoreTestCase(unittest.TestCase): self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler - self.u_alice = hs.parse_userid("@alice:test") - self.u_bob = hs.parse_userid("@bob:test") + self.u_alice = UserID.from_string("@alice:test") + self.u_bob = UserID.from_string("@bob:test") # User elsewhere on another host - self.u_charlie = hs.parse_userid("@charlie:elsewhere") + self.u_charlie = UserID.from_string("@charlie:elsewhere") self.room = hs.parse_roomid("!abc123:test") diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 9247fc579e..d7c7f64d5e 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -48,8 +49,8 @@ class StreamStoreTestCase(unittest.TestCase): self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler - self.u_alice = hs.parse_userid("@alice:test") - self.u_bob = hs.parse_userid("@bob:test") + self.u_alice = UserID.from_string("@alice:test") + self.u_bob = UserID.from_string("@bob:test") self.room1 = hs.parse_roomid("!abc123:test") self.room2 = hs.parse_roomid("!xyx987:test") diff --git a/tests/test_types.py b/tests/test_types.py index bfb9e6f548..2de7f22ab0 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -42,12 +42,6 @@ class UserIDTestCase(unittest.TestCase): self.assertTrue(userA == userAagain) self.assertTrue(userA != userB) - def test_via_homeserver(self): - user = mock_homeserver.parse_userid("@3456ijkl:my.domain") - - self.assertEquals("3456ijkl", user.localpart) - self.assertEquals("my.domain", user.domain) - class RoomAliasTestCase(unittest.TestCase): -- cgit 1.4.1 From 1c06c48ce2db3c6355e29de1533aebf36bc3775b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 11:55:12 +0000 Subject: Replace hs.parse_roomid with RoomID.from_string --- synapse/handlers/room.py | 2 -- synapse/rest/client/v1/room.py | 4 ++-- synapse/server.py | 7 +------ tests/storage/test_directory.py | 3 ++- tests/storage/test_redaction.py | 4 ++-- tests/storage/test_room.py | 6 +++--- tests/storage/test_roommember.py | 4 ++-- tests/storage/test_stream.py | 6 +++--- 8 files changed, 15 insertions(+), 21 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0242288c4e..edb96cec83 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -246,8 +246,6 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def get_room_members(self, room_id): - hs = self.hs - users = yield self.store.get_users_in_room(room_id) defer.returnValue([UserID.from_string(u) for u in users]) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c5837b3403..f0a9c932c1 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -20,7 +20,7 @@ from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID import json import logging @@ -227,7 +227,7 @@ class JoinRoomAliasServlet(RestServlet): identifier = self.hs.parse_roomalias(room_identifier) is_room_alias = True except SynapseError: - identifier = self.hs.parse_roomid(room_identifier) + identifier = RoomID.from_string(room_identifier) # TODO: Support for specifying the home server to join with? diff --git a/synapse/server.py b/synapse/server.py index 52a21aaf78..4dfff04277 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -26,7 +26,7 @@ from synapse.api.auth import Auth from synapse.handlers import Handlers from synapse.state import StateHandler from synapse.storage import DataStore -from synapse.types import UserID, RoomAlias, RoomID, EventID +from synapse.types import RoomAlias, EventID from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager @@ -132,11 +132,6 @@ class BaseHomeServer(object): object.""" return RoomAlias.from_string(s) - def parse_roomid(self, s): - """Parse the string given by 's' as a Room ID and return a RoomID - object.""" - return RoomID.from_string(s) - def parse_eventid(self, s): """Parse the string given by 's' as a Event ID and return a EventID object.""" diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py index e9c242cc07..1bc6391766 100644 --- a/tests/storage/test_directory.py +++ b/tests/storage/test_directory.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.directory import DirectoryStore +from synapse.types import RoomID from tests.utils import SQLiteMemoryDbPool @@ -37,7 +38,7 @@ class DirectoryStoreTestCase(unittest.TestCase): self.store = DirectoryStore(hs) - self.room = hs.parse_roomid("!abcde:test") + self.room = RoomID.from_string("!abcde:test") self.alias = hs.parse_roomalias("#my-room:test") @defer.inlineCallbacks diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index a16ccad881..0713dfab64 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -52,7 +52,7 @@ class RedactionTestCase(unittest.TestCase): self.u_alice = UserID.from_string("@alice:test") self.u_bob = UserID.from_string("@bob:test") - self.room1 = hs.parse_roomid("!abc123:test") + self.room1 = RoomID.from_string("!abc123:test") self.depth = 1 diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index c6bfde069a..baec3a3bb9 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool @@ -39,7 +39,7 @@ class RoomStoreTestCase(unittest.TestCase): # management of the 'room_aliases' table self.store = hs.get_datastore() - self.room = hs.parse_roomid("!abcde:test") + self.room = RoomID.from_string("!abcde:test") self.alias = hs.parse_roomalias("#a-room-name:test") self.u_creator = UserID.from_string("@creator:test") @@ -98,7 +98,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): self.store = hs.get_datastore() self.event_factory = hs.get_event_factory(); - self.room = hs.parse_roomid("!abcde:test") + self.room = RoomID.from_string("!abcde:test") yield self.store.store_room(self.room.to_string(), room_creator_user_id="@creator:text", diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 6b7930b1d8..2b9048e2a9 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -56,7 +56,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): # User elsewhere on another host self.u_charlie = UserID.from_string("@charlie:elsewhere") - self.room = hs.parse_roomid("!abc123:test") + self.room = RoomID.from_string("!abc123:test") @defer.inlineCallbacks def inject_room_member(self, room, user, membership, replaces_state=None): diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index d7c7f64d5e..b7f6e2aa80 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -52,8 +52,8 @@ class StreamStoreTestCase(unittest.TestCase): self.u_alice = UserID.from_string("@alice:test") self.u_bob = UserID.from_string("@bob:test") - self.room1 = hs.parse_roomid("!abc123:test") - self.room2 = hs.parse_roomid("!xyx987:test") + self.room1 = RoomID.from_string("!abc123:test") + self.room2 = RoomID.from_string("!xyx987:test") self.depth = 1 -- cgit 1.4.1 From ada711504efb4dd25fe1123d38a0b2d196b9890a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 13:21:58 +0000 Subject: Replace hs.parse_roomalias with RoomAlias.from_string --- synapse/handlers/directory.py | 3 ++- synapse/rest/client/v1/directory.py | 9 +++++---- synapse/rest/client/v1/room.py | 4 ++-- synapse/server.py | 7 +------ tests/handlers/test_directory.py | 7 ++++--- tests/storage/test_directory.py | 4 ++-- tests/storage/test_room.py | 4 ++-- tests/test_types.py | 6 ------ 8 files changed, 18 insertions(+), 26 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 91fceda2ac..58e9a91562 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -19,6 +19,7 @@ from ._base import BaseHandler from synapse.api.errors import SynapseError, Codes, CodeMessageException from synapse.api.constants import EventTypes +from synapse.types import RoomAlias import logging @@ -122,7 +123,7 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def on_directory_query(self, args): - room_alias = self.hs.parse_roomalias(args["room_alias"]) + room_alias = RoomAlias.from_string(args["room_alias"]) if not self.hs.is_mine(room_alias): raise SynapseError( 400, "Room Alias is not hosted on this Home Server" diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 7ff44fdd9e..1f33ec9e81 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -17,7 +17,8 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError, Codes -from base import RestServlet, client_path_pattern +from synapse.types import RoomAlias +from .base import RestServlet, client_path_pattern import json import logging @@ -35,7 +36,7 @@ class ClientDirectoryServer(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_alias): - room_alias = self.hs.parse_roomalias(room_alias) + room_alias = RoomAlias.from_string(room_alias) dir_handler = self.handlers.directory_handler res = yield dir_handler.get_association(room_alias) @@ -53,7 +54,7 @@ class ClientDirectoryServer(RestServlet): logger.debug("Got content: %s", content) - room_alias = self.hs.parse_roomalias(room_alias) + room_alias = RoomAlias.from_string(room_alias) logger.debug("Got room name: %s", room_alias.to_string()) @@ -92,7 +93,7 @@ class ClientDirectoryServer(RestServlet): dir_handler = self.handlers.directory_handler - room_alias = self.hs.parse_roomalias(room_alias) + room_alias = RoomAlias.from_string(room_alias) yield dir_handler.delete_association( user.to_string(), room_alias diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f0a9c932c1..42712d4a7c 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -20,7 +20,7 @@ from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID, RoomID +from synapse.types import UserID, RoomID, RoomAlias import json import logging @@ -224,7 +224,7 @@ class JoinRoomAliasServlet(RestServlet): identifier = None is_room_alias = False try: - identifier = self.hs.parse_roomalias(room_identifier) + identifier = RoomAlias.from_string(room_identifier) is_room_alias = True except SynapseError: identifier = RoomID.from_string(room_identifier) diff --git a/synapse/server.py b/synapse/server.py index 4dfff04277..41a26ad91a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -26,7 +26,7 @@ from synapse.api.auth import Auth from synapse.handlers import Handlers from synapse.state import StateHandler from synapse.storage import DataStore -from synapse.types import RoomAlias, EventID +from synapse.types import EventID from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager @@ -127,11 +127,6 @@ class BaseHomeServer(object): # TODO: Why are these parse_ methods so high up along with other globals? # Surely these should be in a util package or in the api package? - def parse_roomalias(self, s): - """Parse the string given by 's' as a Room Alias and return a RoomAlias - object.""" - return RoomAlias.from_string(s) - def parse_eventid(self, s): """Parse the string given by 's' as a Event ID and return a EventID object.""" diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 8e164e4be0..22119de46a 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -21,6 +21,7 @@ from mock import Mock from synapse.server import HomeServer from synapse.handlers.directory import DirectoryHandler +from synapse.types import RoomAlias from tests.utils import SQLiteMemoryDbPool, MockKey @@ -65,9 +66,9 @@ class DirectoryTestCase(unittest.TestCase): self.store = hs.get_datastore() - self.my_room = hs.parse_roomalias("#my-room:test") - self.your_room = hs.parse_roomalias("#your-room:test") - self.remote_room = hs.parse_roomalias("#another:remote") + self.my_room = RoomAlias.from_string("#my-room:test") + self.your_room = RoomAlias.from_string("#your-room:test") + self.remote_room = RoomAlias.from_string("#another:remote") @defer.inlineCallbacks def test_get_local_association(self): diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py index 1bc6391766..bc9ebf35e2 100644 --- a/tests/storage/test_directory.py +++ b/tests/storage/test_directory.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.directory import DirectoryStore -from synapse.types import RoomID +from synapse.types import RoomID, RoomAlias from tests.utils import SQLiteMemoryDbPool @@ -39,7 +39,7 @@ class DirectoryStoreTestCase(unittest.TestCase): self.store = DirectoryStore(hs) self.room = RoomID.from_string("!abcde:test") - self.alias = hs.parse_roomalias("#my-room:test") + self.alias = RoomAlias.from_string("#my-room:test") @defer.inlineCallbacks def test_room_to_alias(self): diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index baec3a3bb9..71e5d34143 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes -from synapse.types import UserID, RoomID +from synapse.types import UserID, RoomID, RoomAlias from tests.utils import SQLiteMemoryDbPool @@ -40,7 +40,7 @@ class RoomStoreTestCase(unittest.TestCase): self.store = hs.get_datastore() self.room = RoomID.from_string("!abcde:test") - self.alias = hs.parse_roomalias("#a-room-name:test") + self.alias = RoomAlias.from_string("#a-room-name:test") self.u_creator = UserID.from_string("@creator:test") yield self.store.store_room(self.room.to_string(), diff --git a/tests/test_types.py b/tests/test_types.py index 2de7f22ab0..b29a8415b1 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -56,9 +56,3 @@ class RoomAliasTestCase(unittest.TestCase): room = RoomAlias("channel", "my.domain") self.assertEquals(room.to_string(), "#channel:my.domain") - - def test_via_homeserver(self): - room = mock_homeserver.parse_roomalias("#elsewhere:my.domain") - - self.assertEquals("elsewhere", room.localpart) - self.assertEquals("my.domain", room.domain) -- cgit 1.4.1 From 0cfb4591a7754bcc08edddd17629006b5096d94d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 15:46:31 +0000 Subject: Add handler for /sync API --- synapse/handlers/sync.py | 110 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 synapse/handlers/sync.py (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py new file mode 100644 index 0000000000..ec20ea4890 --- /dev/null +++ b/synapse/handlers/sync.py @@ -0,0 +1,110 @@ +import collections + + +SyncConfig = collections.namedtuple("SyncConfig", [ + "user", + "device", + "since", + "limit", + "gap", + "sort" + "backfill" + "filter", +) + + +RoomSyncResult = collections.namedtuple("RoomSyncResult", [ + "room_id", + "limited", + "published", + "prev_batch", + "events", + "state", + "event_map", +]) + + +class SyncResult(collections.namedtuple("SyncResult", [ + "next_batch", # Token for the next sync + "private_user_data", # List of private events for the user. + "public_user_data", # List of public events for all users. + "rooms", # RoomSyncResult for each room. +])): + __slots__ = [] + + def __nonzero__(self): + return self.private_user_data or self.public_user_data or self.rooms + + +class SyncHandler(BaseHandler): + + def __init__(self, hs): + super(SyncHandler, self).__init__(hs) + self.event_sources = hs.get_event_sources() + + def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): + if timeout == 0: + return self.current_sync_for_user(sync_config, since) + else: + def current_sync_callback(since_token): + return self.current_sync_for_user( + self, since_token, sync_config + ) + return self.notifier.wait_for_events( + sync_config.filter, since_token, current_sync_callback + ) + defer.returnValue(result) + + def current_sync_for_user(self, sync_config, since_token=None): + if since_token is None: + return self.inital_sync(sync_config) + else: + return self.incremental_sync(sync_config) + + @defer.inlineCallbacks + def initial_sync(self, sync_config): + now_token = yield self.event_sources.get_current_token() + + presence_stream = self.event_sources.sources["presence"] + # TODO (markjh): This looks wrong, shouldn't we be getting the presence + # UP to the present rather than after the present? + pagination_config = PaginationConfig(from_token=now_token) + presence, _ = yield presence_stream.get_pagination_rows( + user, pagination_config.get_source_config("presence"), None + ) + room_list = yield self.store.get_rooms_for_user_where_membership_is( + user_id=user_id, + membership_list=[Membership.INVITE, Membership.JOIN] + ) + + # TODO (markjh): Does public mean "published"? + published_rooms = yield self.store.get_rooms(is_public=True) + published_room_ids = set(r["room_id"] for r in public_rooms) + + for event in room_list: + + messages, token = yield self.store.get_recent_events_for_room( + event.room_id, + limit=sync_config.limit, + end_token=now_token.room_key, + ) + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state = yield self.state_handler.get_current_state( + event.room_id + ) + + rooms.append(RoomSyncResult( + room_id=event.room_id, + published=event.room_id in published_room_ids, + + + + + + @defer.inlineCallbacks + def incremental_sync(self, sync_config): + + + + + -- cgit 1.4.1 From 7f6f3f9d6247076493c4e9d48c8282e25892f8b5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 16:11:28 +0000 Subject: Pass the current time to serialize event, rather than passing an HS and getting a clock from it and calling time_msec on the clock. Remove the serialize_event method from the HS since it is no longer needed. --- synapse/events/utils.py | 12 ++++++------ synapse/handlers/events.py | 5 ++++- synapse/handlers/message.py | 21 +++++++++++++++------ synapse/handlers/room.py | 7 +++++-- synapse/rest/client/v1/events.py | 8 +++++++- synapse/rest/client/v1/room.py | 9 ++++++++- synapse/server.py | 3 --- 7 files changed, 45 insertions(+), 20 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index bcb5457278..e391aca4cc 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,31 +89,31 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(hs, e, client_event=True): +def serialize_event(e, time_now_ms, client_event=True): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e + time_now_ms = int(time_now_ms) + # Should this strip out None's? d = {k: v for k, v in e.get_dict().items()} if not client_event: # set the age and keep all other keys if "age_ts" in d["unsigned"]: - now = int(hs.get_clock().time_msec()) - d["unsigned"]["age"] = now - d["unsigned"]["age_ts"] + d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] return d if "age_ts" in d["unsigned"]: - now = int(hs.get_clock().time_msec()) - d["age"] = now - d["unsigned"]["age_ts"] + d["age"] = time_now_ms - d["unsigned"]["age_ts"] del d["unsigned"]["age_ts"] d["user_id"] = d.pop("sender", None) if "redacted_because" in e.unsigned: d["redacted_because"] = serialize_event( - hs, e.unsigned["redacted_because"] + e.unsigned["redacted_because"], time_now_ms ) del d["unsigned"]["redacted_because"] diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 01e67b0818..d997917cd6 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logcontext import PreserveLoggingContext from synapse.util.logutils import log_function from synapse.types import UserID +from synapse.events.utils import serialize_event from ._base import BaseHandler @@ -78,8 +79,10 @@ class EventStreamHandler(BaseHandler): auth_user, room_ids, pagin_config, timeout ) + time_now = self.clock.time_msec() + chunks = [ - self.hs.serialize_event(e, as_client_event) for e in events + serialize_event(e, time_now, as_client_event) for e in events ] chunk = { diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6a1104a890..9c3271fe88 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig +from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator from synapse.util.logcontext import PreserveLoggingContext from synapse.types import UserID @@ -100,9 +101,11 @@ class MessageHandler(BaseHandler): "room_key", next_key ) + time_now = self.clock.time_msec() + chunk = { "chunk": [ - self.hs.serialize_event(e, as_client_event) for e in events + serialize_event(e, time_now, as_client_event) for e in events ], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), @@ -211,7 +214,8 @@ class MessageHandler(BaseHandler): # TODO: This is duplicating logic from snapshot_all_rooms current_state = yield self.state_handler.get_current_state(room_id) - defer.returnValue([self.hs.serialize_event(c) for c in current_state]) + now = self.clock.time_msec() + defer.returnValue([serialize_event(c, now) for c in current_state]) @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, @@ -283,10 +287,11 @@ class MessageHandler(BaseHandler): start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) + time_now = self.clock.time_msec() d["messages"] = { "chunk": [ - self.hs.serialize_event(m, as_client_event) + serialize_event(m, time_now, as_client_event) for m in messages ], "start": start_token.to_string(), @@ -297,7 +302,8 @@ class MessageHandler(BaseHandler): event.room_id ) d["state"] = [ - self.hs.serialize_event(c) for c in current_state + serialize_event(c, time_now, as_client_event) + for c in current_state ] except: logger.exception("Failed to get snapshot") @@ -320,8 +326,9 @@ class MessageHandler(BaseHandler): auth_user = UserID.from_string(user_id) # TODO: These concurrently + time_now = self.clock.time_msec() state_tuples = yield self.state_handler.get_current_state(room_id) - state = [self.hs.serialize_event(x) for x in state_tuples] + state = [serialize_event(x, time_now) for x in state_tuples] member_event = (yield self.store.get_room_member( user_id=user_id, @@ -360,11 +367,13 @@ class MessageHandler(BaseHandler): "Failed to get member presence of %r", m.user_id ) + time_now = self.clock.time_msec() + defer.returnValue({ "membership": member_event.membership, "room_id": room_id, "messages": { - "chunk": [self.hs.serialize_event(m) for m in messages], + "chunk": [serialize_event(m, time_now) for m in messages], "start": start_token.to_string(), "end": end_token.to_string(), }, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index edb96cec83..23821d321f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -16,12 +16,14 @@ """Contains functions for performing events on rooms.""" from twisted.internet import defer +from ._base import BaseHandler + from synapse.types import UserID, RoomAlias, RoomID from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import StoreError, SynapseError from synapse.util import stringutils from synapse.util.async import run_on_reactor -from ._base import BaseHandler +from synapse.events.utils import serialize_event import logging @@ -293,8 +295,9 @@ class RoomMemberHandler(BaseHandler): yield self.auth.check_joined_room(room_id, user_id) member_list = yield self.store.get_room_members(room_id=room_id) + time_now = self.clock.time_msec() event_list = [ - self.hs.serialize_event(entry) + serialize_event(entry, time_now) for entry in member_list ] chunk_data = { diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index c69de56863..a0d051227b 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig from .base import ClientV1RestServlet, client_path_pattern +from synapse.events.utils import serialize_event import logging @@ -64,14 +65,19 @@ class EventStreamRestServlet(ClientV1RestServlet): class EventRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/events/(?P[^/]*)$") + def __init__(self, hs): + super(EventRestServlet, self).__init__(hs) + self.clock = hs.get_clock() + @defer.inlineCallbacks def on_GET(self, request, event_id): auth_user = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) + time_now = self.clock.time_msec() if event: - defer.returnValue((200, self.hs.serialize_event(event))) + defer.returnValue((200, serialize_event(event, time_now))) else: defer.returnValue((404, "Event not found.")) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f06e3ddb98..58b09b6fc1 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -21,6 +21,7 @@ from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID, RoomAlias +from synapse.events.utils import serialize_event import json import logging @@ -363,6 +364,10 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): class RoomTriggerBackfill(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/backfill$") + def __init__(self, hs): + super(RoomTriggerBackfill, self).__init__(hs) + self.clock = hs.get_clock() + @defer.inlineCallbacks def on_GET(self, request, room_id): remote_server = urllib.unquote( @@ -374,7 +379,9 @@ class RoomTriggerBackfill(ClientV1RestServlet): handler = self.handlers.federation_handler events = yield handler.backfill(remote_server, room_id, limit) - res = [self.hs.serialize_event(event) for event in events] + time_now = self.clock.time_msec() + + res = [serialize_event(event, time_now) for event in events] defer.returnValue((200, res)) diff --git a/synapse/server.py b/synapse/server.py index 92ed2c5e32..c478f812e6 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -124,9 +124,6 @@ class BaseHomeServer(object): setattr(BaseHomeServer, "get_%s" % (depname), _get) - def serialize_event(self, e, as_client_event=True): - return serialize_event(self, e, as_client_event) - def get_ip_from_request(self, request): # May be an X-Forwarding-For header depending on config ip_addr = request.getClientIP() -- cgit 1.4.1 From 436513068de73ab47d9ba9a32046420be3d86588 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 18:53:31 +0000 Subject: Start implementing the non-incremental sync portion of the v2 /sync API --- synapse/events/utils.py | 6 +- synapse/handlers/__init__.py | 2 + synapse/handlers/sync.py | 87 ++++++++++++++++++++--------- synapse/rest/client/v2_alpha/sync.py | 105 +++++++++++++++++++++++++---------- 4 files changed, 146 insertions(+), 54 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index e391aca4cc..b7f1ad4b40 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,7 +89,7 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(e, time_now_ms, client_event=True): +def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e @@ -138,4 +138,8 @@ def serialize_event(e, time_now_ms, client_event=True): d.pop("unsigned", None) d.pop("origin", None) + if strip_ids: + d.pop("room_id", None) + d.pop("event_id", None) + return d diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index fe071a4bc2..a32eab9316 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -26,6 +26,7 @@ from .presence import PresenceHandler from .directory import DirectoryHandler from .typing import TypingNotificationHandler from .admin import AdminHandler +from .sync import SyncHandler class Handlers(object): @@ -51,3 +52,4 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) + self.sync_handler = SyncHandler(hs) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ec20ea4890..bbabaf3df1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1,26 +1,49 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import BaseHandler + +from synapse.streams.config import PaginationConfig +from synapse.api.constants import Membership + +from twisted.internet import defer + import collections +import logging + +logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", "device", - "since", "limit", "gap", - "sort" - "backfill" + "sort", + "backfill", "filter", -) +]) RoomSyncResult = collections.namedtuple("RoomSyncResult", [ "room_id", "limited", "published", - "prev_batch", - "events", + "events", # dict of event "state", - "event_map", + "prev_batch", ]) @@ -41,10 +64,11 @@ class SyncHandler(BaseHandler): def __init__(self, hs): super(SyncHandler, self).__init__(hs) self.event_sources = hs.get_event_sources() + self.clock = hs.get_clock() def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): if timeout == 0: - return self.current_sync_for_user(sync_config, since) + return self.current_sync_for_user(sync_config, since_token) else: def current_sync_callback(since_token): return self.current_sync_for_user( @@ -53,58 +77,71 @@ class SyncHandler(BaseHandler): return self.notifier.wait_for_events( sync_config.filter, since_token, current_sync_callback ) - defer.returnValue(result) def current_sync_for_user(self, sync_config, since_token=None): if since_token is None: - return self.inital_sync(sync_config) + return self.initial_sync(sync_config) else: return self.incremental_sync(sync_config) @defer.inlineCallbacks def initial_sync(self, sync_config): + if sync_config.sort == "timeline,desc": + # TODO(mjark): Handle going through events in reverse order?. + # What does "most recent events" mean when applying the limits mean + # in this case? + raise NotImplementedError() + now_token = yield self.event_sources.get_current_token() presence_stream = self.event_sources.sources["presence"] - # TODO (markjh): This looks wrong, shouldn't we be getting the presence + # TODO (mjark): This looks wrong, shouldn't we be getting the presence # UP to the present rather than after the present? pagination_config = PaginationConfig(from_token=now_token) presence, _ = yield presence_stream.get_pagination_rows( - user, pagination_config.get_source_config("presence"), None + user=sync_config.user, + pagination_config=pagination_config.get_source_config("presence"), + key=None ) room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=user_id, + user_id=sync_config.user.to_string(), membership_list=[Membership.INVITE, Membership.JOIN] ) - # TODO (markjh): Does public mean "published"? + # TODO (mjark): Does public mean "published"? published_rooms = yield self.store.get_rooms(is_public=True) - published_room_ids = set(r["room_id"] for r in public_rooms) + published_room_ids = set(r["room_id"] for r in published_rooms) + rooms = [] for event in room_list: - - messages, token = yield self.store.get_recent_events_for_room( + #TODO (mjark): Apply the event filter in sync_config. + recent_events, token = yield self.store.get_recent_events_for_room( event.room_id, limit=sync_config.limit, end_token=now_token.room_key, ) prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - current_state = yield self.state_handler.get_current_state( + current_state_events = yield self.state_handler.get_current_state( event.room_id ) rooms.append(RoomSyncResult( room_id=event.room_id, published=event.room_id in published_room_ids, + events=recent_events, + prev_batch=prev_batch_token, + state=current_state_events, + limited=True, + )) - - + defer.returnValue(SyncResult( + public_user_data=presence, + private_user_data=[], + rooms=rooms, + next_batch=now_token, + )) @defer.inlineCallbacks def incremental_sync(self, sync_config): - - - - - + pass diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 39bb5ec8e9..cc667ebafc 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,9 @@ from twisted.internet import defer from synapse.http.servlet import RestServlet +from synapse.handlers.sync import SyncConfig +from synapse.types import StreamToken +from synapse.events.utils import serialize_event from ._base import client_v2_pattern import logging @@ -73,14 +76,15 @@ class SyncRestServlet(RestServlet): def __init__(self, hs): super(SyncRestServlet, self).__init__() self.auth = hs.get_auth() - #self.sync_handler = hs.get_handlers().sync_hanlder + self.sync_handler = hs.get_handlers().sync_handler + self.clock = hs.get_clock() @defer.inlineCallbacks def on_GET(self, request): user = yield self.auth.get_user_by_req(request) timeout = self.parse_integer(request, "timeout", default=0) - limit = self.parse_integer(request, "limit", default=None) + limit = self.parse_integer(request, "limit", required=True) gap = self.parse_boolean(request, "gap", default=True) sort = self.parse_string( request, "sort", default="timeline,asc", @@ -91,7 +95,7 @@ class SyncRestServlet(RestServlet): request, "set_presence", default="online", allowed_values=self.ALLOWED_PRESENCE ) - backfill = self.parse_boolean(request, "backfill", default=True) + backfill = self.parse_boolean(request, "backfill", default=False) filter_id = self.parse_string(request, "filter", default=None) logger.info( @@ -108,36 +112,81 @@ class SyncRestServlet(RestServlet): # if filter.matches(event): # # stuff - # if timeout != 0: - # register for updates from the event stream - - #rooms = [] - - if gap: - pass - # now_stream_token = get_current_stream_token - # for room_id in get_rooms_for_user(user, filter=filter): - # state, events, start, end, limited, published = updates_for_room( - # from=since, to=now_stream_token, limit=limit, - # anchor_to_start=False - # ) - # rooms[room_id] = (state, events, start, limited, published) - # next_stream_token = now. + sync_config = SyncConfig( + user=user, + device="TODO", # TODO(mjark) Get the device_id from access_token + gap=gap, + limit=limit, + sort=sort, + backfill=backfill, + filter="TODO", # TODO(mjark) Add the filter to the config. + ) + + if since is not None: + since_token = StreamToken.from_string(since) else: - pass - # now_stream_token = get_current_stream_token - # for room_id in get_rooms_for_user(user, filter=filter) - # state, events, start, end, limited, published = updates_for_room( - # from=since, to=now_stream_token, limit=limit, - # anchor_to_start=False - # ) - # next_stream_token = min(next_stream_token, end) + since_token = None + sync_result = yield self.sync_handler.wait_for_sync_for_user( + sync_config, since_token=since_token, timeout=timeout + ) - response_content = {} + time_now = self.clock.time_msec() + + response_content = { + "public_user_data": self.encode_events( + sync_result.public_user_data, filter, time_now + ), + "private_user_data": self.encode_events( + sync_result.private_user_data, filter, time_now + ), + "rooms": self.encode_rooms(sync_result.rooms, filter, time_now), + "next_batch": sync_result.next_batch.to_string(), + } defer.returnValue((200, response_content)) + def encode_events(self, events, filter, time_now): + return [self.encode_event(event, filter, time_now) for event in events] + + @staticmethod + def encode_event(event, filter, time_now): + # TODO(mjark): Respect formatting requirements in the filter. + return serialize_event(event, time_now) + + def encode_rooms(self, rooms, filter, time_now): + return [self.encode_room(room, filter, time_now) for room in rooms] + + @staticmethod + def encode_room(room, filter, time_now): + event_map = {} + state_event_ids = [] + recent_event_ids = [] + for event in room.state: + # TODO(mjark): Respect formatting requirements in the filter. + event_map[event.event_id] = serialize_event( + event, time_now, strip_ids=True + ) + state_event_ids.append(event.event_id) + + for event in room.events: + # TODO(mjark): Respect formatting requirements in the filter. + event_map[event.event_id] = serialize_event( + event, time_now, strip_ids=True + ) + recent_event_ids.append(event.event_id) + return { + "room_id": room.room_id, + "event_map": event_map, + "events": { + "batch": recent_event_ids, + "prev_batch": room.prev_batch.to_string(), + }, + "state": state_event_ids, + "limited": room.limited, + "published": room.published, + } + def register_servlets(hs, http_server): SyncRestServlet(hs).register(http_server) -- cgit 1.4.1 From 51449e06654c4af7a645124dc64e1f0cc1678b24 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 15:50:28 +0000 Subject: Add appservice handler and store. Glue together rest > handler > store. --- synapse/handlers/__init__.py | 2 ++ synapse/handlers/appservice.py | 49 ++++++++++++++++++++++++++++++++++ synapse/rest/appservice/v1/base.py | 3 ++- synapse/rest/appservice/v1/register.py | 8 ++++-- synapse/storage/__init__.py | 5 ++-- synapse/storage/appservice.py | 45 +++++++++++++++++++++++++++++++ 6 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 synapse/handlers/appservice.py create mode 100644 synapse/storage/appservice.py (limited to 'synapse/handlers') diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index fe071a4bc2..96a9b143ca 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -26,6 +26,7 @@ from .presence import PresenceHandler from .directory import DirectoryHandler from .typing import TypingNotificationHandler from .admin import AdminHandler +from .appservice import ApplicationServicesHandler class Handlers(object): @@ -51,3 +52,4 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) + self.appservice_handler = ApplicationServicesHandler(hs) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py new file mode 100644 index 0000000000..55a653476f --- /dev/null +++ b/synapse/handlers/appservice.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import BaseHandler +from synapse.api.errors import StoreError, SynapseError + +import logging + + +logger = logging.getLogger(__name__) + + +class ApplicationServicesHandler(BaseHandler): + + def __init__(self, hs): + super(ApplicationServicesHandler, self).__init__(hs) + + @defer.inlineCallbacks + def register(self, base_url, token, namespaces): + # check the token is recognised + try: + app_service = yield self.store.get_app_service(token) + if not app_service: + raise StoreError + except StoreError: + raise SynapseError( + 403, "Unrecognised application services token. " + "Consult the home server admin." + ) + + # update AS entry with base URL + + # store namespaces for this AS + + defer.returnValue("not_implemented_yet") diff --git a/synapse/rest/appservice/v1/base.py b/synapse/rest/appservice/v1/base.py index 46c9a444c0..65d5bcf9be 100644 --- a/synapse/rest/appservice/v1/base.py +++ b/synapse/rest/appservice/v1/base.py @@ -44,4 +44,5 @@ class AppServiceRestServlet(RestServlet): """ def __init__(self, hs): - self.hs = hs \ No newline at end of file + self.hs = hs + self.handler = hs.get_handlers().appservice_handler diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 779447ac6a..142f09a638 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -14,6 +14,7 @@ # limitations under the License. """This module contains REST servlets to do with registration: /register""" +from twisted.internet import defer from base import AppServiceRestServlet, as_path_pattern from synapse.api.errors import CodeMessageException, SynapseError @@ -30,6 +31,7 @@ class RegisterRestServlet(AppServiceRestServlet): PATTERN = as_path_pattern("/register$") + @defer.inlineCallbacks def on_POST(self, request): params = _parse_json(request) @@ -56,9 +58,11 @@ class RegisterRestServlet(AppServiceRestServlet): self._parse_namespace(namespaces, params["namespaces"], "rooms") self._parse_namespace(namespaces, params["namespaces"], "aliases") - # TODO: pass to the appservice handler + hs_token = yield self.handler.register(as_url, as_token, namespaces) - raise CodeMessageException(500, "Not implemented.") + defer.returnValue({ + "hs_token": hs_token + }) def _parse_namespace(self, target_ns, origin_ns, ns): if ns not in target_ns or ns not in origin_ns: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4beb951b9f..9431c1a32d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.api.constants import EventTypes +from .appservice import ApplicationServiceStore from .directory import DirectoryStore from .feedback import FeedbackStore from .presence import PresenceStore @@ -80,8 +81,8 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, - EventFederationStore, - MediaRepositoryStore, + EventFederationStore, MediaRepositoryStore, + ApplicationServiceStore ): def __init__(self, hs): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py new file mode 100644 index 0000000000..99f58b4c62 --- /dev/null +++ b/synapse/storage/appservice.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import StoreError + +from ._base import SQLBaseStore + + +class ApplicationServiceStore(SQLBaseStore): + + def __init__(self, hs): + super(ApplicationServiceStore, self).__init__(hs) + + self.clock = hs.get_clock() + + @defer.inlineCallbacks + def get_app_service(self, as_token): + """Get the application service with the given token. + + Args: + token (str): The application service token. + Raises: + StoreError if there was a problem retrieving this. + """ + row = self._simple_select_one( + "application_services", {"token": as_token}, + ["url", "token"] + ) + if not row: + raise StoreError(400, "Bad application services token supplied.") + defer.returnValue(row) -- cgit 1.4.1 From 1d779691248be2fcdd43e97352d13c2cef239c10 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 15:58:27 +0000 Subject: Unbreak bad presence merge - don't add these blocks together with an and: they're different things. --- synapse/handlers/events.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 75fb941008..f1a3e4a4ad 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -51,19 +51,21 @@ class EventStreamHandler(BaseHandler): auth_user = self.hs.parse_userid(auth_user_id) try: - if affect_presence and auth_user not in self._streams_per_user: - self._streams_per_user[auth_user] = 0 - if auth_user in self._stop_timer_per_user: - try: - self.clock.cancel_call_later( - self._stop_timer_per_user.pop(auth_user) + if affect_presence: + if auth_user not in self._streams_per_user: + self._streams_per_user[auth_user] = 0 + if auth_user in self._stop_timer_per_user: + try: + print "cancel",auth_user + self.clock.cancel_call_later( + self._stop_timer_per_user.pop(auth_user) + ) + except: + logger.exception("Failed to cancel event timer") + else: + yield self.distributor.fire( + "started_user_eventstream", auth_user ) - except: - logger.exception("Failed to cancel event timer") - else: - yield self.distributor.fire( - "started_user_eventstream", auth_user - ) self._streams_per_user[auth_user] += 1 if pagin_config.from_token is None: -- cgit 1.4.1 From eba89f093f0376b8499953b53bbd85f9b12b7852 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:00:07 +0000 Subject: Need a defer.inlineCallbacks here as we yield in it: otherwise nothing in the cb gets executed. --- synapse/handlers/events.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse/handlers') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index f1a3e4a4ad..851eebf600 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -99,6 +99,7 @@ class EventStreamHandler(BaseHandler): # 10 seconds of grace to allow the client to reconnect again # before we think they're gone + @defer.inlineCallbacks def _later(): logger.debug( "_later stopped_user_eventstream %s", auth_user -- cgit 1.4.1 From 5eacaeb4a73aa4c4ff450d7b23acabe79124171f Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:05:23 +0000 Subject: or of course we could just return the deferred --- synapse/handlers/events.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 851eebf600..48de3630e3 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -99,7 +99,6 @@ class EventStreamHandler(BaseHandler): # 10 seconds of grace to allow the client to reconnect again # before we think they're gone - @defer.inlineCallbacks def _later(): logger.debug( "_later stopped_user_eventstream %s", auth_user @@ -107,7 +106,7 @@ class EventStreamHandler(BaseHandler): self._stop_timer_per_user.pop(auth_user, None) - yield self.distributor.fire( + return self.distributor.fire( "stopped_user_eventstream", auth_user ) -- cgit 1.4.1 From f7c4daa8f94e5968277ba438f6d3da1f0f27ba4c Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:08:47 +0000 Subject: Oops, remove debugging --- synapse/handlers/events.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 48de3630e3..52ef07eaae 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -56,7 +56,6 @@ class EventStreamHandler(BaseHandler): self._streams_per_user[auth_user] = 0 if auth_user in self._stop_timer_per_user: try: - print "cancel",auth_user self.clock.cancel_call_later( self._stop_timer_per_user.pop(auth_user) ) -- cgit 1.4.1 From a56008842b43089433768f569f35b2d14523ac39 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 16:24:22 +0000 Subject: Start implementing incremental initial sync --- synapse/events/utils.py | 1 + synapse/handlers/sync.py | 233 +++++++++++++++++++++++++++++++++++++++++----- synapse/storage/stream.py | 41 ++++++-- 3 files changed, 241 insertions(+), 34 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index b7f1ad4b40..42fb0371e5 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -137,6 +137,7 @@ def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): d.pop("depth", None) d.pop("unsigned", None) d.pop("origin", None) + d.pop("prev_state", None) if strip_ids: d.pop("room_id", None) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index bbabaf3df1..f8629a588f 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -37,14 +37,18 @@ SyncConfig = collections.namedtuple("SyncConfig", [ ]) -RoomSyncResult = collections.namedtuple("RoomSyncResult", [ +class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "room_id", "limited", "published", - "events", # dict of event + "events", "state", "prev_batch", -]) +])): + __slots__ = [] + + def __nonzero__(self): + return bool(self.events or self.state) class SyncResult(collections.namedtuple("SyncResult", [ @@ -56,7 +60,9 @@ class SyncResult(collections.namedtuple("SyncResult", [ __slots__ = [] def __nonzero__(self): - return self.private_user_data or self.public_user_data or self.rooms + return bool( + self.private_user_data or self.public_user_data or self.rooms + ) class SyncHandler(BaseHandler): @@ -67,7 +73,13 @@ class SyncHandler(BaseHandler): self.clock = hs.get_clock() def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): - if timeout == 0: + """Get the sync for a client if we have new data for it now. Otherwise + wait for new data to arrive on the server. If the timeout expires, then + return an empty sync result. + Returns: + A Deferred SyncResult. + """ + if timeout == 0 or since_token is None: return self.current_sync_for_user(sync_config, since_token) else: def current_sync_callback(since_token): @@ -79,13 +91,25 @@ class SyncHandler(BaseHandler): ) def current_sync_for_user(self, sync_config, since_token=None): + """Get the sync for client needed to match what the server has now. + Returns: + A Deferred SyncResult. + """ if since_token is None: return self.initial_sync(sync_config) else: - return self.incremental_sync(sync_config) + if sync_config.gap: + return self.incremental_sync_with_gap(sync_config, since_token) + else: + #TODO(mjark): Handle gapless sync + pass @defer.inlineCallbacks def initial_sync(self, sync_config): + """Get a sync for a client which is starting without any state + Returns: + A Deferred SyncResult. + """ if sync_config.sort == "timeline,desc": # TODO(mjark): Handle going through events in reverse order?. # What does "most recent events" mean when applying the limits mean @@ -114,25 +138,86 @@ class SyncHandler(BaseHandler): rooms = [] for event in room_list: - #TODO (mjark): Apply the event filter in sync_config. - recent_events, token = yield self.store.get_recent_events_for_room( - event.room_id, - limit=sync_config.limit, - end_token=now_token.room_key, - ) - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - current_state_events = yield self.state_handler.get_current_state( - event.room_id + room_sync = yield self.initial_sync_for_room( + event.room_id, sync_config, now_token, published_room_ids ) + rooms.append(room_sync) + + defer.returnValue(SyncResult( + public_user_data=presence, + private_user_data=[], + rooms=rooms, + next_batch=now_token, + )) + + @defer.inlineCallbacks + def intial_sync_for_room(self, room_id, sync_config, now_token, + published_room_ids): + """Sync a room for a client which is starting without any state + Returns: + A Deferred RoomSyncResult. + """ + recent_events, token = yield self.store.get_recent_events_for_room( + room_id, + limit=sync_config.limit, + end_token=now_token.room_key, + ) + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state_events = yield self.state_handler.get_current_state( + room_id + ) + + defer.returnValue(RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recent_events, + prev_batch=prev_batch_token, + state=current_state_events, + limited=True, + )) + + + @defer.inlineCallbacks + def incremental_sync_with_gap(self, sync_config, since_token): + """ Get the incremental delta needed to bring the client up to + date with the server. + Returns: + A Deferred SyncResult. + """ + if sync_config.sort == "timeline,desc": + # TODO(mjark): Handle going through events in reverse order?. + # What does "most recent events" mean when applying the limits mean + # in this case? + raise NotImplementedError() + + now_token = yield self.event_sources.get_current_token() + + presence_stream = self.event_sources.sources["presence"] + pagination_config = PaginationConfig( + from_token=since_token, to_token=now_token + ) + presence, _ = yield presence_stream.get_pagination_rows( + user=sync_config.user, + pagination_config=pagination_config.get_source_config("presence"), + key=None + ) + room_list = yield self.store.get_rooms_for_user_where_membership_is( + user_id=sync_config.user.to_string(), + membership_list=[Membership.INVITE, Membership.JOIN] + ) + + # TODO (mjark): Does public mean "published"? + published_rooms = yield self.store.get_rooms(is_public=True) + published_room_ids = set(r["room_id"] for r in published_rooms) - rooms.append(RoomSyncResult( - room_id=event.room_id, - published=event.room_id in published_room_ids, - events=recent_events, - prev_batch=prev_batch_token, - state=current_state_events, - limited=True, - )) + rooms = [] + for event in room_list: + room_sync = yield self.incremental_sync_with_gap_for_room( + event.room_id, sync_config, since_token, now_token, + published_room_ids + ) + if room_sync: + rooms.append(room_sync) defer.returnValue(SyncResult( public_user_data=presence, @@ -143,5 +228,103 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks - def incremental_sync(self, sync_config): - pass + def incremental_sync_with_gap_for_room(self, room_id, sync_config, + since_token, now_token, + published_room_ids): + """ Get the incremental delta needed to bring the client up to date for + the room. Gives the client the most recent events and the changes to + state. + Returns: + A Deferred RoomSyncResult + """ + # TODO(mjark): Check if they have joined the room between + # the previous sync and this one. + # TODO(mjark): Apply the event filter in sync_config + # TODO(mjark): Check for redactions we might have missed. + # TODO(mjark): Typing notifications. + recents, token = yield self.store.get_recent_events_for_room( + room_id, + limit=sync_config.limit + 1, + from_token=since_token.room_key, + end_token=now_token.room_key, + ) + + logging.debug("Recents %r", recents) + + if len(recents) > sync_config.limit: + limited = True + recents = recents[1:] + else: + limited = False + + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + + # TODO(mjark): This seems racy since this isn't being passed a + # token to indicate what point in the stream this is + current_state_events = yield self.state_handler.get_current_state( + room_id + ) + + state_at_previous_sync = yield self.get_state_at_previous_sync( + room_id, since_token=since_token + ) + + state_events_delta = yield self.compute_state_delta( + since_token=since_token, + previous_state=state_at_previous_sync, + current_state=current_state_events, + ) + + room_sync = RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recents, + prev_batch=prev_batch_token, + state=state_events_delta, + limited=limited, + ) + + logging.debug("Room sync: %r", room_sync) + + defer.returnValue(room_sync) + + @defer.inlineCallbacks + def get_state_at_previous_sync(self, room_id, since_token): + """ Get the room state at the previous sync the client made. + Returns: + A Deferred list of Events. + """ + last_events, token = yield self.store.get_recent_events_for_room( + room_id, end_token=since_token.room_key, limit=1, + ) + + if last_events: + last_event = last_events[0] + last_context = yield self.state_handler.compute_event_context( + last_event + ) + if last_event.is_state(): + state = [last_event] + last_context.current_state.values() + else: + state = last_context.current_state.values() + else: + state = () + defer.returnValue(state) + + + def compute_state_delta(self, since_token, previous_state, current_state): + """ Works out the differnce in state between the current state and the + state the client got when it last performed a sync. + Returns: + A list of events. + """ + # TODO(mjark) Check if the state events were received by the server + # after the previous sync, since we need to include those state + # updates even if they occured logically before the previous event. + # TODO(mjark) Check for new redactions in the state events. + previous_dict = {event.event_id:event for event in previous_state} + state_delta = [] + for event in current_state: + if event.event_id not in previous_dict: + state_delta.append(event) + return state_delta diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8ac2adab05..06aca1a4e5 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -265,17 +265,38 @@ class StreamStore(SQLBaseStore): return self.runInteraction("paginate_room_events", f) def get_recent_events_for_room(self, room_id, limit, end_token, - with_feedback=False): + with_feedback=False, from_token=None): # TODO (erikj): Handle compressed feedback - sql = ( - "SELECT stream_ordering, topological_ordering, event_id FROM events " - "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 " - "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " - ) + end_token = _StreamToken.parse_stream_token(end_token) - def f(txn): - txn.execute(sql, (room_id, end_token, limit,)) + if from_token is None: + sql = ( + "SELECT stream_ordering, topological_ordering, event_id" + " FROM events" + " WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0" + " ORDER BY topological_ordering DESC, stream_ordering DESC" + " LIMIT ?" + ) + else: + from_token = _StreamToken.parse_stream_token(from_token) + sql = ( + "SELECT stream_ordering, topological_ordering, event_id" + " FROM events" + " WHERE room_id = ? AND stream_ordering > ?" + " AND stream_ordering <= ? AND outlier = 0" + " ORDER BY topological_ordering DESC, stream_ordering DESC" + " LIMIT ?" + ) + + + def get_recent_events_for_room_txn(txn): + if from_token is None: + txn.execute(sql, (room_id, end_token.stream, limit,)) + else: + txn.execute(sql, ( + room_id, from_token.stream, end_token.stream, limit + )) rows = self.cursor_to_dict(txn) @@ -303,7 +324,9 @@ class StreamStore(SQLBaseStore): return events, token - return self.runInteraction("get_recent_events_for_room", f) + return self.runInteraction( + "get_recent_events_for_room", get_recent_events_for_room_txn + ) def get_room_events_max_id(self): return self.runInteraction( -- cgit 1.4.1 From 92171f9dd1ecac24aeae2f46729f3cbbbe94f91e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 16:53:59 +0000 Subject: Add stub methods, TODOs and docstrings for application services. --- synapse/handlers/appservice.py | 25 +++++++++++++++++--- synapse/storage/appservice.py | 52 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 5 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 55a653476f..25e1cece56 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -42,8 +42,27 @@ class ApplicationServicesHandler(BaseHandler): "Consult the home server admin." ) - # update AS entry with base URL - - # store namespaces for this AS + # store this AS defer.returnValue("not_implemented_yet") + + def unregister(self, token): + yield self.store.unregister_app_service(token) + + def notify_interested_services(self, event): + """Notifies (pushes) all application services interested in this event. + + Pushing is done asynchronously, so this method won't block for any + prolonged length of time. + + Args: + event(Event): The event to push out to interested services. + """ + # TODO: Gather interested services + # get_services_for_event(event) <-- room IDs and user IDs + # Get a list of room aliases. Check regex. + # TODO: If unknown user: poke User Query API. + # TODO: If unknown room alias: poke Room Alias Query API. + + # TODO: Fork off pushes to these services - XXX First cut, best effort + pass diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 4c11191fe8..fbad17cb9e 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -84,16 +84,60 @@ class ApplicationServiceStore(SQLBaseStore): super(ApplicationServiceStore, self).__init__(hs) self.cache = ApplicationServiceCache() self.clock = hs.get_clock() + self._populate_cache() + + def unregister_app_service(self, token): + """Unregisters this service. + + This removes all AS specific regex and the base URL. The token is the + only thing preserved for future registration attempts. + """ + # TODO: DELETE FROM application_services_regex WHERE id=this service + # TODO: SET url=NULL WHERE token=token + # TODO: Update cache + pass + + def update_app_service(self, service): + """Update an application service, clobbering what was previously there. + + Args: + service(ApplicationService): The updated service. + """ + # NB: There is no "insert" since we provide no public-facing API to + # allocate new ASes. It relies on the server admin inserting the AS + # token into the database manually. + + # TODO: UPDATE application_services, SET url WHERE token=service.token + # TODO: DELETE FROM application_services_regex WHERE id=this service + # TODO: INSERT INTO application_services_regex + # TODO: Update cache + pass + + def get_services_for_event(self, event): + return self.cache.get_services_for_event(event) @defer.inlineCallbacks - def get_app_service(self, as_token): + def get_app_service(self, as_token, from_cache=True): """Get the application service with the given token. Args: token (str): The application service token. + from_cache (bool): True to get this service from the cache, False to + check the database. Raises: - StoreError if there was a problem retrieving this. + StoreError if there was a problem retrieving this service. """ + + if from_cache: + for service in self.cache.services: + if service.token == as_token: + defer.returnValue(service) + return + defer.returnValue(None) + return + + + # TODO: This should be JOINed with the application_services_regex table. row = self._simple_select_one( "application_services", {"token": as_token}, ["url", "token"] @@ -101,3 +145,7 @@ class ApplicationServiceStore(SQLBaseStore): if not row: raise StoreError(400, "Bad application services token supplied.") defer.returnValue(row) + + def _populate_cache(self): + """Populates the ApplicationServiceCache from the database.""" + pass -- cgit 1.4.1 From ec3719b583c6fbbc56dbd313b858054e535ae733 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 17:15:06 +0000 Subject: Use ApplicationService when registering. --- synapse/handlers/appservice.py | 13 +++++-------- synapse/rest/appservice/v1/register.py | 8 ++++++-- synapse/storage/appservice.py | 16 ++++++---------- 3 files changed, 17 insertions(+), 20 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 25e1cece56..1890ca06aa 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -30,21 +30,18 @@ class ApplicationServicesHandler(BaseHandler): super(ApplicationServicesHandler, self).__init__(hs) @defer.inlineCallbacks - def register(self, base_url, token, namespaces): + def register(self, app_service): # check the token is recognised try: - app_service = yield self.store.get_app_service(token) - if not app_service: - raise StoreError + stored_service = yield self.store.get_app_service(app_service.token) + if not stored_service: + raise StoreError(404, "Not found") except StoreError: raise SynapseError( 403, "Unrecognised application services token. " "Consult the home server admin." ) - - # store this AS - - defer.returnValue("not_implemented_yet") + # TODO store this AS def unregister(self, token): yield self.store.unregister_app_service(token) diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 142f09a638..5786cf873e 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -18,6 +18,7 @@ from twisted.internet import defer from base import AppServiceRestServlet, as_path_pattern from synapse.api.errors import CodeMessageException, SynapseError +from synapse.storage.appservice import ApplicationService import json import logging @@ -58,7 +59,10 @@ class RegisterRestServlet(AppServiceRestServlet): self._parse_namespace(namespaces, params["namespaces"], "rooms") self._parse_namespace(namespaces, params["namespaces"], "aliases") - hs_token = yield self.handler.register(as_url, as_token, namespaces) + app_service = ApplicationService(as_token, as_url, namespaces) + + yield self.handler.register(app_service) + hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? defer.returnValue({ "hs_token": hs_token @@ -97,7 +101,7 @@ class UnregisterRestServlet(AppServiceRestServlet): except (KeyError, ValueError): raise SynapseError(400, "Missing required key: as_token(str)") - # TODO: pass to the appservice handler + yield self.handler.unregister(as_token) raise CodeMessageException(500, "Not implemented") diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index fbad17cb9e..f84f026b7b 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -116,8 +116,7 @@ class ApplicationServiceStore(SQLBaseStore): def get_services_for_event(self, event): return self.cache.get_services_for_event(event) - @defer.inlineCallbacks - def get_app_service(self, as_token, from_cache=True): + def get_app_service(self, token, from_cache=True): """Get the application service with the given token. Args: @@ -130,21 +129,18 @@ class ApplicationServiceStore(SQLBaseStore): if from_cache: for service in self.cache.services: - if service.token == as_token: - defer.returnValue(service) - return - defer.returnValue(None) - return - + if service.token == token: + return service + return None # TODO: This should be JOINed with the application_services_regex table. row = self._simple_select_one( - "application_services", {"token": as_token}, + "application_services", {"token": token}, ["url", "token"] ) if not row: raise StoreError(400, "Bad application services token supplied.") - defer.returnValue(row) + return row def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" -- cgit 1.4.1 From fbeaeb868960099c3682802275d5a222c0cc2d8b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 17:34:40 +0000 Subject: Log when ASes are registered/unregistered. --- synapse/handlers/appservice.py | 2 ++ synapse/storage/appservice.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 1890ca06aa..c9f56c41eb 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -31,6 +31,7 @@ class ApplicationServicesHandler(BaseHandler): @defer.inlineCallbacks def register(self, app_service): + logger.info("Register -> %s", app_service) # check the token is recognised try: stored_service = yield self.store.get_app_service(app_service.token) @@ -44,6 +45,7 @@ class ApplicationServicesHandler(BaseHandler): # TODO store this AS def unregister(self, token): + logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) def notify_interested_services(self, event): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index f84f026b7b..cd15843ba3 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -51,6 +51,9 @@ class ApplicationService(object): return True + def __str__(self): + return "ApplicationService: %s" % (self.__dict__,) + class ApplicationServiceCache(object): """Caches ApplicationServices and provides utility functions on top. @@ -83,7 +86,6 @@ class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.cache = ApplicationServiceCache() - self.clock = hs.get_clock() self._populate_cache() def unregister_app_service(self, token): -- cgit 1.4.1 From b19cf6a105ad87954e15cf01e60e14fec280db6d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 20:09:52 +0000 Subject: Wait for events if the incremental sync is empty and a timeout is given --- demo/start.sh | 2 +- synapse/handlers/sync.py | 19 ++++++++++++------- synapse/notifier.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 8 deletions(-) (limited to 'synapse/handlers') diff --git a/demo/start.sh b/demo/start.sh index bb2248770d..a7502e7a8d 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -16,7 +16,7 @@ if [ $# -eq 1 ]; then fi fi -for port in 8080 8081 8082; do +for port in 8080; do echo "Starting server on port $port... " https_port=$((port + 400)) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f8629a588f..9f5f73eab6 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -72,6 +72,7 @@ class SyncHandler(BaseHandler): self.event_sources = hs.get_event_sources() self.clock = hs.get_clock() + @defer.inlineCallbacks def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): """Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then @@ -80,15 +81,19 @@ class SyncHandler(BaseHandler): A Deferred SyncResult. """ if timeout == 0 or since_token is None: - return self.current_sync_for_user(sync_config, since_token) + result = yield self.current_sync_for_user(sync_config, since_token) + defer.returnValue(result) else: - def current_sync_callback(since_token): - return self.current_sync_for_user( - self, since_token, sync_config - ) - return self.notifier.wait_for_events( - sync_config.filter, since_token, current_sync_callback + def current_sync_callback(): + return self.current_sync_for_user(sync_config, since_token) + + rm_handler = self.hs.get_handlers().room_member_handler + room_ids = yield rm_handler.get_rooms_for_user(sync_config.user) + result = yield self.notifier.wait_for_events( + sync_config.user, room_ids, + sync_config.filter, timeout, current_sync_callback ) + defer.returnValue(result) def current_sync_for_user(self, sync_config, since_token=None): """Get the sync for client needed to match what the server has now. diff --git a/synapse/notifier.py b/synapse/notifier.py index 3aec1d4af2..922bf064d0 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext from synapse.util.async import run_on_reactor +from synapse.types import StreamToken import logging @@ -205,6 +206,53 @@ class Notifier(object): [notify(l).addErrback(eb) for l in listeners] ) + @defer.inlineCallbacks + def wait_for_events(self, user, rooms, filter, timeout, callback): + """Wait until the callback returns a non empty response or the + timeout fires. + """ + + deferred = defer.Deferred() + + from_token=StreamToken("s0","0","0") + + listener = [_NotificationListener( + user=user, + rooms=rooms, + from_token=from_token, + limit=1, + timeout=timeout, + deferred=deferred, + )] + + if timeout: + self._register_with_keys(listener[0]) + + result = yield callback() + if timeout: + timed_out = [False] + def _timeout_listener(): + timed_out[0] = True + listener[0].notify(self, [], from_token, from_token) + + self.clock.call_later(timeout/1000., _timeout_listener) + while not result and not timed_out[0]: + yield deferred + deferred = defer.Deferred() + listener[0] = _NotificationListener( + user=user, + rooms=rooms, + from_token=from_token, + limit=1, + timeout=timeout, + deferred=deferred, + ) + self._register_with_keys(listener[0]) + result = yield callback() + + defer.returnValue(result) + + def get_events_for(self, user, rooms, pagination_config, timeout): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any -- cgit 1.4.1 From e020574d65a994858ac53c45070ae5016090d2f3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 20:19:36 +0000 Subject: Fix Formatting --- synapse/handlers/sync.py | 13 +++++-------- synapse/notifier.py | 4 ++-- synapse/rest/client/v2_alpha/sync.py | 5 ++--- synapse/storage/stream.py | 1 - 4 files changed, 9 insertions(+), 14 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9f5f73eab6..82a2c6986a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -52,10 +52,10 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ class SyncResult(collections.namedtuple("SyncResult", [ - "next_batch", # Token for the next sync - "private_user_data", # List of private events for the user. - "public_user_data", # List of public events for all users. - "rooms", # RoomSyncResult for each room. + "next_batch", # Token for the next sync + "private_user_data", # List of private events for the user. + "public_user_data", # List of public events for all users. + "rooms", # RoomSyncResult for each room. ])): __slots__ = [] @@ -181,7 +181,6 @@ class SyncHandler(BaseHandler): limited=True, )) - @defer.inlineCallbacks def incremental_sync_with_gap(self, sync_config, since_token): """ Get the incremental delta needed to bring the client up to @@ -231,7 +230,6 @@ class SyncHandler(BaseHandler): next_batch=now_token, )) - @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, @@ -316,7 +314,6 @@ class SyncHandler(BaseHandler): state = () defer.returnValue(state) - def compute_state_delta(self, since_token, previous_state, current_state): """ Works out the differnce in state between the current state and the state the client got when it last performed a sync. @@ -327,7 +324,7 @@ class SyncHandler(BaseHandler): # after the previous sync, since we need to include those state # updates even if they occured logically before the previous event. # TODO(mjark) Check for new redactions in the state events. - previous_dict = {event.event_id:event for event in previous_state} + previous_dict = {event.event_id: event for event in previous_state} state_delta = [] for event in current_state: if event.event_id not in previous_dict: diff --git a/synapse/notifier.py b/synapse/notifier.py index 922bf064d0..e3b6ead620 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -214,7 +214,7 @@ class Notifier(object): deferred = defer.Deferred() - from_token=StreamToken("s0","0","0") + from_token = StreamToken("s0", "0", "0") listener = [_NotificationListener( user=user, @@ -231,6 +231,7 @@ class Notifier(object): result = yield callback() if timeout: timed_out = [False] + def _timeout_listener(): timed_out[0] = True listener[0].notify(self, [], from_token, from_token) @@ -252,7 +253,6 @@ class Notifier(object): defer.returnValue(result) - def get_events_for(self, user, rooms, pagination_config, timeout): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index cc667ebafc..0c17208cd3 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -68,7 +68,6 @@ class SyncRestServlet(RestServlet): } """ - PATTERN = client_v2_pattern("/sync$") ALLOWED_SORT = set(["timeline,asc", "timeline,desc"]) ALLOWED_PRESENCE = set(["online", "offline", "idle"]) @@ -114,12 +113,12 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, - device="TODO", # TODO(mjark) Get the device_id from access_token + device="TODO", # TODO(mjark) Get the device_id from access_token gap=gap, limit=limit, sort=sort, backfill=backfill, - filter="TODO", # TODO(mjark) Add the filter to the config. + filter="TODO", # TODO(mjark) Add the filter to the config. ) if since is not None: diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 06aca1a4e5..db1816ea84 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -289,7 +289,6 @@ class StreamStore(SQLBaseStore): " LIMIT ?" ) - def get_recent_events_for_room_txn(txn): if from_token is None: txn.execute(sql, (room_id, end_token.stream, limit,)) -- cgit 1.4.1 From e32ded7b3e290067f95497ceee0ac04842ba9cdf Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 10:09:54 +0000 Subject: Add matrix.org as a trusted ID server because it's now passed through on ports 80/443 and the web client defaults to that now. Fixes email validation (including signing up with an email address). --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 732652c228..66a89c10b2 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -163,7 +163,7 @@ class RegistrationHandler(BaseHandler): # each request httpCli = SimpleHttpClient(self.hs) # XXX: make this configurable! - trustedIdServers = ['matrix.org:8090'] + trustedIdServers = ['matrix.org:8090', 'matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) -- cgit 1.4.1 From 273b12729b99addf4474c9092f44ff300fd8153b Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 11:55:49 +0000 Subject: Reset badge count to zero when last active time is bumped --- synapse/handlers/presence.py | 5 +++++ synapse/push/__init__.py | 19 +++++++++++++++++++ synapse/push/httppusher.py | 38 +++++++++++++++++++++++++++++++++++--- synapse/push/pusherpool.py | 17 +++++++++++++++++ 4 files changed, 76 insertions(+), 3 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 8aeed99274..24d901b51f 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -86,6 +86,10 @@ class PresenceHandler(BaseHandler): "changed_presencelike_data", self.changed_presencelike_data ) + # outbound signal from the presence module to advertise when a user's + # presence has changed + distributor.declare("user_presence_changed") + self.distributor = distributor self.federation = hs.get_replication_layer() @@ -603,6 +607,7 @@ class PresenceHandler(BaseHandler): room_ids=room_ids, statuscache=statuscache, ) + yield self.distributor.fire("user_presence_changed", user, statuscache) @defer.inlineCallbacks def _push_presence_remote(self, user, destination, state=None): diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index b6d01a82a0..4862d0de27 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -54,6 +54,9 @@ class Pusher(object): self.failing_since = failing_since self.alive = True + # The last value of last_active_time that we saw + self.last_last_active_time = 0 + @defer.inlineCallbacks def _actions_for_event(self, ev): """ @@ -273,6 +276,22 @@ class Pusher(object): """ pass + def reset_badge_count(self): + pass + + def presence_changed(self, state): + """ + We clear badge counts whenever a user's last_active time is bumped + This is by no means perfect but I think it's the best we can do + without read receipts. + """ + if 'last_active' in state.state: + last_active = state.state['last_active'] + if last_active > self.last_last_active_time: + logger.info("Resetting badge count for %s", self.user_name) + self.reset_badge_count() + self.last_last_active_time = last_active + def _value_for_dotted_key(dotted_key, event): parts = dotted_key.split(".") diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 22532fcc6a..d592bc2fd2 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -71,11 +71,12 @@ class HttpPusher(Pusher): # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? #'from_display_name': 'Steve Stevington' - #'counts': { -- we don't mark messages as read yet so + 'counts': { #-- we don't mark messages as read yet so # we have no way of knowing - # 'unread': 1, + # Just set the badge to 1 until we have read receipts + 'unread': 1, # 'missed_calls': 2 - # }, + }, 'devices': [ { 'app_id': self.app_id, @@ -111,3 +112,34 @@ class HttpPusher(Pusher): if 'rejected' in resp: rejected = resp['rejected'] defer.returnValue(rejected) + + @defer.inlineCallbacks + def reset_badge_count(self): + d = { + 'notification': { + 'id': '', + 'type': None, + 'from': '', + 'counts': { + 'unread': 0, + 'missed_calls': 0 + }, + 'devices': [ + { + 'app_id': self.app_id, + 'pushkey': self.pushkey, + 'pushkey_ts': long(self.pushkey_ts / 1000), + 'data': self.data_minus_url, + } + ] + } + } + try: + resp = yield self.httpCli.post_json_get_json(self.url, d) + except: + logger.exception("Failed to push %s ", self.url) + defer.returnValue(False) + rejected = [] + if 'rejected' in resp: + rejected = resp['rejected'] + defer.returnValue(rejected) \ No newline at end of file diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 2dfecf178b..65ab4f46e1 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -18,6 +18,7 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException +from synapse.api.constants import PresenceState import logging import json @@ -32,6 +33,22 @@ class PusherPool: self.pushers = {} self.last_pusher_started = -1 + distributor = self.hs.get_distributor() + distributor.observe( + "user_presence_changed", self.user_presence_changed + ) + + @defer.inlineCallbacks + def user_presence_changed(self, user, state): + user_name = user.to_string() + + # until we have read receipts, pushers use this to reset a user's + # badge counters to zero + for p in self.pushers.values(): + if p.user_name == user_name: + yield p.presence_changed(state) + + @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() -- cgit 1.4.1 From 42876969b99b6bad146b44a734e8d4a1a14d6835 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 11:59:38 +0000 Subject: Add basic application_services SQL, and hook up parts of the appservice store to read from it. --- synapse/handlers/appservice.py | 7 +- synapse/rest/appservice/v1/register.py | 4 +- synapse/storage/__init__.py | 1 + synapse/storage/appservice.py | 89 ++++++++++++++++++++++--- synapse/storage/schema/application_services.sql | 32 +++++++++ 5 files changed, 117 insertions(+), 16 deletions(-) create mode 100644 synapse/storage/schema/application_services.sql (limited to 'synapse/handlers') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index c9f56c41eb..8bd475cbfd 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -16,7 +16,7 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.errors import StoreError, SynapseError +from synapse.api.errors import Codes, StoreError, SynapseError import logging @@ -36,11 +36,12 @@ class ApplicationServicesHandler(BaseHandler): try: stored_service = yield self.store.get_app_service(app_service.token) if not stored_service: - raise StoreError(404, "Not found") + raise StoreError(404, "Application Service Not found") except StoreError: raise SynapseError( 403, "Unrecognised application services token. " - "Consult the home server admin." + "Consult the home server admin.", + errcode=Codes.FORBIDDEN ) # TODO store this AS diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 5786cf873e..e374d538e7 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -64,9 +64,9 @@ class RegisterRestServlet(AppServiceRestServlet): yield self.handler.register(app_service) hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? - defer.returnValue({ + defer.returnValue((200, { "hs_token": hs_token - }) + })) def _parse_namespace(self, target_ns, origin_ns, ns): if ns not in target_ns or ns not in origin_ns: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9431c1a32d..e86b981b47 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -62,6 +62,7 @@ SCHEMAS = [ "event_edges", "event_signatures", "media_repository", + "application_services" ] diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 533fac4972..5a0e47e0d4 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -12,12 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from synapse.api.errors import StoreError +import logging +from twisted.internet import defer from ._base import SQLBaseStore +logger = logging.getLogger(__name__) + + # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): """Defines an application service. @@ -30,7 +33,22 @@ class ApplicationService(object): if url: self.url = url if namespaces: - self.namespaces = namespaces + self._set_namespaces(namespaces) + + def _set_namespaces(self, namespaces): + # Sanity check that it is of the form: + # { + # users: ["regex",...], + # aliases: ["regex",...], + # rooms: ["regex",...], + # } + for ns in ["users", "rooms", "aliases"]: + if type(namespaces[ns]) != list: + raise ValueError("Bad namespace value for '%s'", ns) + for regex in namespaces[ns]: + if not isinstance(regex, basestring): + raise ValueError("Expected string regex for ns '%s'", ns) + self.namespaces = namespaces def is_interested(self, event): """Check if this service is interested in this event. @@ -133,15 +151,64 @@ class ApplicationServiceStore(SQLBaseStore): return service return None + # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. - row = self._simple_select_one( - "application_services", {"token": token}, - ["url", "token"] - ) - if not row: - raise StoreError(400, "Bad application services token supplied.") - return row + + @defer.inlineCallbacks def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" - pass + sql = ("SELECT * FROM application_services LEFT JOIN " + "application_services_regex ON application_services.id = " + "application_services_regex.as_id") + + namespace_enum = [ + "users", # 0 + "aliases", # 1 + "rooms" # 2 + ] + # SQL results in the form: + # [ + # { + # 'regex': "something", + # 'url': "something", + # 'namespace': enum, + # 'as_id': 0, + # 'token': "something", + # 'id': 0 + # } + # ] + services = {} + results = yield self._execute_and_decode(sql) + for res in results: + as_token = res["token"] + if as_token not in services: + # add the service + services[as_token] = { + "url": res["url"], + "token": as_token, + "namespaces": { + "users": [], + "aliases": [], + "rooms": [] + } + } + # add the namespace regex if one exists + ns_int = res["namespace"] + if ns_int is None: + continue + try: + services[as_token]["namespaces"][namespace_enum[ns_int]].append( + res["regex"] + ) + except IndexError: + logger.error("Bad namespace enum '%s'. %s", ns_int, res) + + for service in services.values(): + logger.info("Found application service: %s", service) + self.cache.services.append(ApplicationService( + service["token"], + service["url"], + service["namespaces"] + )) + diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql new file mode 100644 index 0000000000..6d245fc807 --- /dev/null +++ b/synapse/storage/schema/application_services.sql @@ -0,0 +1,32 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT, + token TEXT, + UNIQUE(token) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id INTEGER PRIMARY KEY AUTOINCREMENT, + as_id INTEGER NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex TEXT, + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + + + -- cgit 1.4.1 From 0ef5bfd6a9eaaae14e199997658b3d0006abd854 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 28 Jan 2015 16:16:53 +0000 Subject: Start implementing auth conflict res --- synapse/api/auth.py | 38 +++--- synapse/api/constants.py | 6 + synapse/federation/federation_client.py | 39 ++++++ synapse/handlers/federation.py | 211 ++++++++++++++++++++++++++------ synapse/storage/rejections.py | 10 ++ synapse/storage/schema/im.sql | 1 + 6 files changed, 253 insertions(+), 52 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a342a0e0da..461faa8c78 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -353,9 +353,23 @@ class Auth(object): def add_auth_events(self, builder, context): yield run_on_reactor() - if builder.type == EventTypes.Create: - builder.auth_events = [] - return + auth_ids = self.compute_auth_events(builder, context) + + auth_events_entries = yield self.store.add_event_hashes( + auth_ids + ) + + builder.auth_events = auth_events_entries + + context.auth_events = { + k: v + for k, v in context.current_state.items() + if v.event_id in auth_ids + } + + def compute_auth_events(self, event, context): + if event.type == EventTypes.Create: + return [] auth_ids = [] @@ -368,7 +382,7 @@ class Auth(object): key = (EventTypes.JoinRules, "", ) join_rule_event = context.current_state.get(key) - key = (EventTypes.Member, builder.user_id, ) + key = (EventTypes.Member, event.user_id, ) member_event = context.current_state.get(key) key = (EventTypes.Create, "", ) @@ -382,8 +396,8 @@ class Auth(object): else: is_public = False - if builder.type == EventTypes.Member: - e_type = builder.content["membership"] + if event.type == EventTypes.Member: + e_type = event.content["membership"] if e_type in [Membership.JOIN, Membership.INVITE]: if join_rule_event: auth_ids.append(join_rule_event.event_id) @@ -398,17 +412,7 @@ class Auth(object): if member_event.content["membership"] == Membership.JOIN: auth_ids.append(member_event.event_id) - auth_events_entries = yield self.store.add_event_hashes( - auth_ids - ) - - builder.auth_events = auth_events_entries - - context.auth_events = { - k: v - for k, v in context.current_state.items() - if v.event_id in auth_ids - } + return auth_ids @log_function def _can_send_event(self, event, auth_events): diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 7ee6dcc46e..0d3fc629af 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -74,3 +74,9 @@ class EventTypes(object): Message = "m.room.message" Topic = "m.room.topic" Name = "m.room.name" + + +class RejectedReason(object): + AUTH_ERROR = "auth_error" + REPLACED = "replaced" + NOT_ANCESTOR = "not_ancestor" diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 91b44cd8b3..ebcd593506 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -331,6 +331,45 @@ class FederationClient(object): defer.returnValue(pdu) + @defer.inlineCallbacks + def query_auth(self, destination, room_id, event_id, local_auth): + """ + Params: + destination (str) + event_it (str) + local_auth (list) + """ + time_now = self._clock.time_msec() + + send_content = { + "auth_chain": [e.get_pdu_json(time_now) for e in local_auth], + } + + code, content = yield self.transport_layer.send_invite( + destination=destination, + room_id=room_id, + event_id=event_id, + content=send_content, + ) + + auth_chain = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content["auth_chain"] + ] + + missing = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content.get("missing", []) + ] + + ret = { + "auth_chain": auth_chain, + "rejects": content.get("rejects", []), + "missing": missing, + } + + defer.returnValue(ret) + def event_from_pdu_json(self, pdu_json, outlier=False): event = FrozenEvent( pdu_json diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index bcdcc90a18..97e3c503b9 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,19 +17,16 @@ from ._base import BaseHandler -from synapse.events.utils import prune_event from synapse.api.errors import ( - AuthError, FederationError, SynapseError, StoreError, + AuthError, FederationError, StoreError, ) -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, RejectedReason from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import ( - compute_event_signature, check_event_content_hash, - add_hashes_and_signatures, + compute_event_signature, add_hashes_and_signatures, ) from synapse.types import UserID -from syutil.jsonutil import encode_canonical_json from twisted.internet import defer @@ -113,33 +110,6 @@ class FederationHandler(BaseHandler): logger.debug("Processing event: %s", event.event_id) - redacted_event = prune_event(event) - - redacted_pdu_json = redacted_event.get_pdu_json() - try: - yield self.keyring.verify_json_for_server( - event.origin, redacted_pdu_json - ) - except SynapseError as e: - logger.warn( - "Signature check failed for %s redacted to %s", - encode_canonical_json(pdu.get_pdu_json()), - encode_canonical_json(redacted_pdu_json), - ) - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event.event_id, - ) - - if not check_event_content_hash(event): - logger.warn( - "Event content has been tampered, redacting %s, %s", - event.event_id, encode_canonical_json(event.get_dict()) - ) - event = redacted_event - logger.debug("Event: %s", event) # FIXME (erikj): Awful hack to make the case where we are not currently @@ -180,7 +150,6 @@ class FederationHandler(BaseHandler): if state: for e in state: - logging.info("A :) %r", e) e.internal_metadata.outlier = True try: yield self._handle_new_event(e) @@ -747,7 +716,20 @@ class FederationHandler(BaseHandler): event.event_id, event.signatures, ) - self.auth.check(event, auth_events=context.auth_events) + try: + self.auth.check(event, auth_events=context.auth_events) + except AuthError: + # TODO: Store rejection. + context.rejected = RejectedReason.AUTH_ERROR + + yield self.store.persist_event( + event, + context=context, + backfilled=backfilled, + is_new_state=False, + current_state=current_state, + ) + raise logger.debug( "_handle_new_event: Before persist_event: %s, sigs: %s", @@ -768,3 +750,162 @@ class FederationHandler(BaseHandler): ) defer.returnValue(context) + + @defer.inlineCallbacks + def do_auth(self, origin, event, context): + for e_id, _ in event.auth_events: + pass + + auth_events = set(e_id for e_id, _ in event.auth_events) + current_state = set(e.event_id for e in context.auth_events.values()) + + missing_auth = auth_events - current_state + + if missing_auth: + # Do auth conflict res. + + # 1. Get what we think is the auth chain. + auth_ids = self.auth.compute_auth_events(event, context) + local_auth_chain = yield self.store.get_auth_chain(auth_ids) + + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) + + # 3. Process any remote auth chain events we haven't seen. + for e in result.get("missing", []): + # TODO. + pass + + # 4. Look at rejects and their proofs. + # TODO. + + try: + self.auth.check(event, auth_events=context.auth_events) + except AuthError: + raise + + @defer.inlineCallbacks + def construct_auth_difference(self, local_auth, remote_auth): + """ Given a local and remote auth chain, find the differences. This + assumes that we have already processed all events in remote_auth + + Params: + local_auth (list) + remote_auth (list) + + Returns: + dict + """ + + # TODO: Make sure we are OK with local_auth or remote_auth having more + # auth events in them than strictly necessary. + + def sort_fun(ev): + return ev.depth, ev.event_id + + # We find the differences by starting at the "bottom" of each list + # and iterating up on both lists. The lists are ordered by depth and + # then event_id, we iterate up both lists until we find the event ids + # don't match. Then we look at depth/event_id to see which side is + # missing that event, and iterate only up that list. Repeat. + + remote_list = list(remote_auth) + remote_list.sort(key=sort_fun) + + local_list = list(local_auth) + local_list.sort(key=sort_fun) + + local_iter = iter(local_list) + remote_iter = iter(remote_list) + + current_local = local_iter.next() + current_remote = remote_iter.next() + + def get_next(it, opt=None): + return it.next() if it.has_next() else opt + + missing_remotes = [] + missing_locals = [] + while current_local and current_remote: + if current_remote is None: + missing_locals.append(current_local) + current_local = get_next(local_iter) + continue + + if current_local is None: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + if current_local.event_id == current_remote.event_id: + current_local = get_next(local_iter) + current_remote = get_next(remote_iter) + continue + + if current_local.depth < current_remote.depth: + missing_locals.append(current_local) + current_local = get_next(local_iter) + continue + + if current_local.depth > current_remote.depth: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + # They have the same depth, so we fall back to the event_id order + if current_local.event_id < current_remote.event_id: + missing_locals.append(current_local) + current_local = get_next(local_iter) + + if current_local.event_id > current_remote.event_id: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + # missing locals should be sent to the server + # We should find why we are missing remotes, as they will have been + # rejected. + + # Remove events from missing_remotes if they are referencing a missing + # remote. We only care about the "root" rejected ones. + missing_remote_ids = [e.event_id for e in missing_remotes] + base_remote_rejected = list(missing_remotes) + for e in missing_remotes: + for e_id, _ in e.auth_events: + if e_id in missing_remote_ids: + base_remote_rejected.remove(e) + + reason_map = {} + + for e in base_remote_rejected: + reason = yield self.store.get_rejection_reason(e.event_id) + if reason is None: + # FIXME: ERRR?! + raise RuntimeError("") + + reason_map[e.event_id] = reason + + if reason == RejectedReason.AUTH_ERROR: + pass + elif reason == RejectedReason.REPLACED: + # TODO: Get proof + pass + elif reason == RejectedReason.NOT_ANCESTOR: + # TODO: Get proof. + pass + + defer.returnValue({ + "rejects": { + e.event_id: { + "reason": reason_map[e.event_id], + "proof": None, + } + for e in base_remote_rejected + }, + "missing": missing_locals, + }) diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py index 7d38b31f44..b7249700d7 100644 --- a/synapse/storage/rejections.py +++ b/synapse/storage/rejections.py @@ -31,3 +31,13 @@ class RejectionsStore(SQLBaseStore): "last_failure": self._clock.time_msec(), } ) + + def get_rejection_reason(self, event_id): + self._simple_select_one_onecol( + table="rejections", + retcol="reason", + keyvalues={ + "event_id": event_id, + }, + allow_none=True, + ) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index bc7c6b6ed5..5866a387f6 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -128,5 +128,6 @@ CREATE TABLE IF NOT EXISTS rejections( event_id TEXT NOT NULL, reason TEXT NOT NULL, last_check TEXT NOT NULL, + root_rejected TEXT, CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE ); -- cgit 1.4.1 From 388581e087a3658c1b70d2aa1d17a132953350ca Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 16:58:23 +0000 Subject: Extract the id token of the token when authing users, include the token and device_id in the internal meta data for the event along with the transaction id when sending events --- synapse/api/auth.py | 8 ++-- synapse/handlers/message.py | 12 +++++- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/directory.py | 4 +- synapse/rest/client/v1/events.py | 4 +- synapse/rest/client/v1/initial_sync.py | 2 +- synapse/rest/client/v1/presence.py | 8 ++-- synapse/rest/client/v1/profile.py | 4 +- synapse/rest/client/v1/room.py | 64 +++++++++++++++++------------ synapse/rest/client/v1/voip.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/rest/media/v1/upload_resource.py | 2 +- synapse/storage/registration.py | 3 +- synapse/types.py | 3 ++ tests/rest/client/v1/test_presence.py | 2 + tests/rest/client/v1/test_rooms.py | 7 ++++ tests/rest/client/v1/test_typing.py | 1 + tests/storage/test_registration.py | 10 ++++- 18 files changed, 92 insertions(+), 48 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 292e9e2a80..3959e06a8b 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,7 +21,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from synapse.types import UserID +from synapse.types import UserID, ClientID import logging @@ -292,7 +292,7 @@ class Auth(object): Returns: Tuple of UserID and device string: User ID object of the user making the request - Device ID string of the device the user is using + Client ID object of the client instance the user is using Raises: AuthError if no user by that token exists or the token is invalid. """ @@ -302,6 +302,7 @@ class Auth(object): user_info = yield self.get_user_by_token(access_token) user = user_info["user"] device_id = user_info["device_id"] + token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders( @@ -317,7 +318,7 @@ class Auth(object): user_agent=user_agent ) - defer.returnValue((user, device_id)) + defer.returnValue((user, ClientID(device_id, token_id))) except KeyError: raise AuthError(403, "Missing access token.") @@ -342,6 +343,7 @@ class Auth(object): "admin": bool(ret.get("admin", False)), "device_id": ret.get("device_id"), "user": UserID.from_string(ret.get("name")), + "token_id": ret.get("token_id", None), } defer.returnValue(user_info) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9c3271fe88..6fbd2af4ab 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -114,7 +114,8 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def create_and_send_event(self, event_dict, ratelimit=True): + def create_and_send_event(self, event_dict, ratelimit=True, + client=None, txn_id=None): """ Given a dict from a client, create and handle a new event. Creates an FrozenEvent object, filling out auth_events, prev_events, @@ -148,6 +149,15 @@ class MessageHandler(BaseHandler): builder.content ) + if client is not None: + if client.token_id is not None: + builder.internal_metadata.token_id = client.token_id + if client.device_id is not None: + builder.internal_metadata.device_id = client.device_id + + if txn_id is not None: + builder.internal_metadata.txn_id = txn_id + event, context = yield self._create_new_client_event( builder=builder, ) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 6cfce1a479..2ce754b028 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -31,7 +31,7 @@ class WhoisRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) if not is_admin and target_user != auth_user: diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index ef853af411..8f65efec5f 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -45,7 +45,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) if not "room_id" in content: @@ -85,7 +85,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request, room_alias): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index e58ee46fcd..77b7c25a03 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -34,7 +34,7 @@ class EventStreamRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) try: handler = self.handlers.event_stream_handler pagin_config = PaginationConfig.from_request(request) @@ -71,7 +71,7 @@ class EventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, event_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 78d30abbf8..4a259bba64 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -25,7 +25,7 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 74669274a7..7feb4aadb1 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -32,7 +32,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( @@ -42,7 +42,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = {} @@ -77,7 +77,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): @@ -97,7 +97,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index f04abb2c26..15d6f3fc6c 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -37,7 +37,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: @@ -70,7 +70,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c8c34b4801..410f19ccf6 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -62,7 +62,7 @@ class RoomCreateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) room_config = self.get_room_config(request) info = yield self.make_room(room_config, auth_user, None) @@ -125,7 +125,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) msg_handler = self.handlers.message_handler data = yield msg_handler.get_room_data( @@ -142,8 +142,8 @@ class RoomStateEventRestServlet(ClientV1RestServlet): defer.returnValue((200, data.get_dict()["content"])) @defer.inlineCallbacks - def on_PUT(self, request, room_id, event_type, state_key): - user, device_id = yield self.auth.get_user_by_req(request) + def on_PUT(self, request, room_id, event_type, state_key, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -158,7 +158,9 @@ class RoomStateEventRestServlet(ClientV1RestServlet): event_dict["state_key"] = state_key msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event(event_dict) + yield msg_handler.create_and_send_event( + event_dict, client=client, txn_id=txn_id, + ) defer.returnValue((200, {})) @@ -172,8 +174,8 @@ class RoomSendEventRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server, with_get=True) @defer.inlineCallbacks - def on_POST(self, request, room_id, event_type): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, event_type, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -183,7 +185,9 @@ class RoomSendEventRestServlet(ClientV1RestServlet): "content": content, "room_id": room_id, "sender": user.to_string(), - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"event_id": event.event_id})) @@ -200,7 +204,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, event_type) + response = yield self.on_POST(request, room_id, event_type, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -215,8 +219,8 @@ class JoinRoomAliasServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_identifier): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_identifier, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) # the identifier could be a room alias or a room id. Try one then the # other if it fails to parse, without swallowing other valid @@ -245,7 +249,9 @@ class JoinRoomAliasServlet(ClientV1RestServlet): "room_id": identifier.to_string(), "sender": user.to_string(), "state_key": user.to_string(), - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"room_id": identifier.to_string()})) @@ -259,7 +265,7 @@ class JoinRoomAliasServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_identifier) + response = yield self.on_POST(request, room_identifier, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -283,7 +289,7 @@ class RoomMemberListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.room_member_handler members = yield handler.get_room_members_as_pagination_chunk( room_id=room_id, @@ -311,7 +317,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request( request, default_limit=10, ) @@ -335,7 +341,7 @@ class RoomStateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.message_handler # Get all the current state for this room events = yield handler.get_state_events( @@ -351,7 +357,7 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, @@ -395,8 +401,8 @@ class RoomMembershipRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_id, membership_action): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, membership_action, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -418,7 +424,9 @@ class RoomMembershipRestServlet(ClientV1RestServlet): "room_id": room_id, "sender": user.to_string(), "state_key": state_key, - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {})) @@ -432,7 +440,9 @@ class RoomMembershipRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, membership_action) + response = yield self.on_POST( + request, room_id, membership_action, txn_id + ) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -444,8 +454,8 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_id, event_id): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, event_id, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -456,7 +466,9 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): "room_id": room_id, "sender": user.to_string(), "redacts": event_id, - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"event_id": event.event_id})) @@ -470,7 +482,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, event_id) + response = yield self.on_POST(request, room_id, event_id, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -483,7 +495,7 @@ class RoomTypingRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) target_user = UserID.from_string(urllib.unquote(user_id)) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 42d8e30bab..11d08fbced 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -28,7 +28,7 @@ class VoipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) turnUris = self.hs.config.turn_uris turnSecret = self.hs.config.turn_shared_secret diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index 311ab89edb..22e26e3cd5 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -66,7 +66,7 @@ class ContentRepoResource(resource.Resource): @defer.inlineCallbacks def map_request_to_name(self, request): # auth the user - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) # namespace all file uploads on the user prefix = base64.urlsafe_b64encode( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 6bed8a8efa..b939a30e19 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -42,7 +42,7 @@ class UploadResource(BaseMediaResource): @defer.inlineCallbacks def _async_render_POST(self, request): try: - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 75dffa4db2..029b07cc66 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -122,7 +122,8 @@ class RegistrationStore(SQLBaseStore): def _query_for_auth(self, txn, token): sql = ( - "SELECT users.name, users.admin, access_tokens.device_id" + "SELECT users.name, users.admin," + " access_tokens.device_id, access_tokens.id as token_id" " FROM users" " INNER JOIN access_tokens on users.id = access_tokens.user_id" " WHERE token = ?" diff --git a/synapse/types.py b/synapse/types.py index faac729ff2..46dbab5374 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -119,3 +119,6 @@ class StreamToken( d = self._asdict() d[key] = new_value return StreamToken(**d) + + +ClientID = namedtuple("ClientID", ("device_id", "token_id")) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index a4f2abf213..f849120a3e 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -75,6 +75,7 @@ class PresenceStateTestCase(unittest.TestCase): "user": UserID.from_string(myid), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -165,6 +166,7 @@ class PresenceListTestCase(unittest.TestCase): "user": UserID.from_string(myid), "admin": False, "device_id": None, + "token_id": 1, } hs.handlers.room_member_handler = Mock( diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 76ed550b75..81ead10e76 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -70,6 +70,7 @@ class RoomPermissionsTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -466,6 +467,7 @@ class RoomsMemberListTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -555,6 +557,7 @@ class RoomsCreateTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -657,6 +660,7 @@ class RoomTopicTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -773,6 +777,7 @@ class RoomMemberStateTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -909,6 +914,7 @@ class RoomMessagesTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -1013,6 +1019,7 @@ class RoomInitialSyncTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index c89b37d004..c5d5b06da3 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -73,6 +73,7 @@ class RoomTypingTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 84bfde7568..6f8bea2f61 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -53,7 +53,10 @@ class RegistrationStoreTestCase(unittest.TestCase): ) self.assertEquals( - {"admin": 0, "device_id": None, "name": self.user_id}, + {"admin": 0, + "device_id": None, + "name": self.user_id, + "token_id": 1}, (yield self.store.get_user_by_token(self.tokens[0])) ) @@ -63,7 +66,10 @@ class RegistrationStoreTestCase(unittest.TestCase): yield self.store.add_access_token_to_user(self.user_id, self.tokens[1]) self.assertEquals( - {"admin": 0, "device_id": None, "name": self.user_id}, + {"admin": 0, + "device_id": None, + "name": self.user_id, + "token_id": 2}, (yield self.store.get_user_by_token(self.tokens[1])) ) -- cgit 1.4.1 From 3dbce6f4a59fde2a67e563ce338f510feda2dd1a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 03:33:51 +0000 Subject: Add typing notifications to sync --- synapse/handlers/sync.py | 30 +++++++++++++++++++++--------- synapse/rest/client/v2_alpha/sync.py | 18 ++++++++---------- 2 files changed, 29 insertions(+), 19 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 82a2c6986a..b86e783e14 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -44,11 +44,12 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "events", "state", "prev_batch", + "typing", ])): __slots__ = [] def __nonzero__(self): - return bool(self.events or self.state) + return bool(self.events or self.state or self.typing) class SyncResult(collections.namedtuple("SyncResult", [ @@ -196,15 +197,25 @@ class SyncHandler(BaseHandler): now_token = yield self.event_sources.get_current_token() - presence_stream = self.event_sources.sources["presence"] - pagination_config = PaginationConfig( - from_token=since_token, to_token=now_token + presence_source = self.event_sources.sources["presence"] + presence, presence_key = yield presence_source.get_new_events_for_user( + user=sync_config.user, + from_key=since_token.presence_key, + limit=sync_config.limit, ) - presence, _ = yield presence_stream.get_pagination_rows( + now_token = now_token.copy_and_replace("presence_key", presence_key) + + typing_source = self.event_sources.sources["typing"] + typing, typing_key = yield typing_source.get_new_events_for_user( user=sync_config.user, - pagination_config=pagination_config.get_source_config("presence"), - key=None + from_key=since_token.typing_key, + limit=sync_config.limit, ) + now_token = now_token.copy_and_replace("typing_key", typing_key) + + typing_by_room = {event["room_id"]: event for event in typing} + logger.debug("Typing %r", typing_by_room) + room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=sync_config.user.to_string(), membership_list=[Membership.INVITE, Membership.JOIN] @@ -218,7 +229,7 @@ class SyncHandler(BaseHandler): for event in room_list: room_sync = yield self.incremental_sync_with_gap_for_room( event.room_id, sync_config, since_token, now_token, - published_room_ids + published_room_ids, typing_by_room ) if room_sync: rooms.append(room_sync) @@ -233,7 +244,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, - published_room_ids): + published_room_ids, typing_by_room): """ Get the incremental delta needed to bring the client up to date for the room. Gives the client the most recent events and the changes to state. @@ -285,6 +296,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, + typing=typing_by_room.get(room_id, None) ) logging.debug("Room sync: %r", room_sync) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 4d950f9956..76489e27c8 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -135,10 +135,10 @@ class SyncRestServlet(RestServlet): time_now = self.clock.time_msec() response_content = { - "public_user_data": self.encode_events( + "public_user_data": self.encode_user_data( sync_result.public_user_data, filter, time_now ), - "private_user_data": self.encode_events( + "private_user_data": self.encode_user_data( sync_result.private_user_data, filter, time_now ), "rooms": self.encode_rooms( @@ -149,13 +149,8 @@ class SyncRestServlet(RestServlet): defer.returnValue((200, response_content)) - def encode_events(self, events, filter, time_now): - return [self.encode_event(event, filter, time_now) for event in events] - - @staticmethod - def encode_event(event, filter, time_now): - # TODO(mjark): Respect formatting requirements in the filter. - return serialize_event(event, time_now) + def encode_user_data(self, events, filter, time_now): + return events def encode_rooms(self, rooms, filter, time_now, token_id): return [ @@ -183,7 +178,7 @@ class SyncRestServlet(RestServlet): event_format=format_event_for_client_v2_without_event_id, ) recent_event_ids.append(event.event_id) - return { + result = { "room_id": room.room_id, "event_map": event_map, "events": { @@ -194,6 +189,9 @@ class SyncRestServlet(RestServlet): "limited": room.limited, "published": room.published, } + if room.typing is not None: + result["typing"] = room.typing + return result def register_servlets(hs, http_server): -- cgit 1.4.1 From e3e72b8c5c004c403929406fc952141fd0f1b8ae Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 03:35:25 +0000 Subject: Remove typing TODO --- synapse/handlers/sync.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b86e783e14..9e1188da56 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -255,7 +255,6 @@ class SyncHandler(BaseHandler): # the previous sync and this one. # TODO(mjark): Apply the event filter in sync_config # TODO(mjark): Check for redactions we might have missed. - # TODO(mjark): Typing notifications. recents, token = yield self.store.get_recent_events_for_room( room_id, limit=sync_config.limit + 1, -- cgit 1.4.1 From e016f4043b81ffdedf71c4459772f66757386e44 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 14:40:28 +0000 Subject: Use get_room_events_stream to get changes to the rooms if the number of changes is small --- synapse/handlers/sync.py | 56 +++++++++++++++++++++++++++++++++++++---------- synapse/storage/stream.py | 7 ++++++ 2 files changed, 52 insertions(+), 11 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9e1188da56..e93dfe005d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -216,23 +216,57 @@ class SyncHandler(BaseHandler): typing_by_room = {event["room_id"]: event for event in typing} logger.debug("Typing %r", typing_by_room) - room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=sync_config.user.to_string(), - membership_list=[Membership.INVITE, Membership.JOIN] - ) + rm_handler = self.hs.get_handlers().room_member_handler + room_ids = yield rm_handler.get_rooms_for_user(sync_config.user) # TODO (mjark): Does public mean "published"? published_rooms = yield self.store.get_rooms(is_public=True) published_room_ids = set(r["room_id"] for r in published_rooms) + room_events, _ = yield self.store.get_room_events_stream( + sync_config.user.to_string(), + from_key=since_token.room_key, + to_key=now_token.room_key, + room_id=None, + limit=sync_config.limit + 1, + ) + rooms = [] - for event in room_list: - room_sync = yield self.incremental_sync_with_gap_for_room( - event.room_id, sync_config, since_token, now_token, - published_room_ids, typing_by_room - ) - if room_sync: - rooms.append(room_sync) + if len(room_events) <= sync_config.limit: + # There is no gap in any of the rooms. Therefore we can just + # partition the new events by room and return them. + events_by_room_id = {} + for event in room_events: + events_by_room_id.setdefault(event.room_id, []).append(event) + + for room_id in room_ids: + recents = events_by_room_id.get(room_id, []) + state = [event for event in recents if event.is_state()] + if recents: + prev_batch = now_token.copy_and_replace( + "room_key", recents[0].internal_metadata.before + ) + else: + prev_batch = now_token + room_sync = RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recents, + prev_batch=prev_batch, + state=state, + limited=False, + typing=typing_by_room.get(room_id, None) + ) + if room_sync is not None: + rooms.append(room_sync) + else: + for room_id in room_ids: + room_sync = yield self.incremental_sync_with_gap_for_room( + room_id, sync_config, since_token, now_token, + published_room_ids, typing_by_room + ) + if room_sync: + rooms.append(room_sync) defer.returnValue(SyncResult( public_user_data=presence, diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index db1816ea84..93ccfd8c10 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -181,6 +181,13 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + for event, row in zip(ret, rows): + stream = row["stream_ordering"] + topo = event.depth + internal = event.internal_metadata + internal.before = str(_StreamToken(topo, stream - 1)) + internal.after = str(_StreamToken(topo, stream)) + if rows: key = "s%d" % max([r["stream_ordering"] for r in rows]) else: -- cgit 1.4.1 From 4d9dd9bdc02ca7b524c90ae527687e7a838acee4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:23:03 +0000 Subject: Fix v2 initial sync --- synapse/handlers/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index e93dfe005d..cfe2944917 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -157,7 +157,7 @@ class SyncHandler(BaseHandler): )) @defer.inlineCallbacks - def intial_sync_for_room(self, room_id, sync_config, now_token, + def initial_sync_for_room(self, room_id, sync_config, now_token, published_room_ids): """Sync a room for a client which is starting without any state Returns: @@ -180,6 +180,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=current_state_events, limited=True, + typing=None, )) @defer.inlineCallbacks -- cgit 1.4.1 From cc42d3f9076db1815a989bab7ef6004796b31b3c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:27:38 +0000 Subject: Fix check for empty room update --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index cfe2944917..ec8beb4c6a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -258,7 +258,7 @@ class SyncHandler(BaseHandler): limited=False, typing=typing_by_room.get(room_id, None) ) - if room_sync is not None: + if room_sync: rooms.append(room_sync) else: for room_id in room_ids: -- cgit 1.4.1 From 722b65f46131349c5afdcc7eb48297cdd9d9cbd6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:41:21 +0000 Subject: Move typing notifs to an "emphermal" event list on the room object --- synapse/handlers/sync.py | 12 +++++++----- synapse/rest/client/v2_alpha/sync.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ec8beb4c6a..3860c3c95b 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -44,12 +44,12 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "events", "state", "prev_batch", - "typing", + "ephemeral", ])): __slots__ = [] def __nonzero__(self): - return bool(self.events or self.state or self.typing) + return bool(self.events or self.state or self.ephemeral) class SyncResult(collections.namedtuple("SyncResult", [ @@ -180,7 +180,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=current_state_events, limited=True, - typing=None, + ephemeral=[], )) @defer.inlineCallbacks @@ -214,7 +214,9 @@ class SyncHandler(BaseHandler): ) now_token = now_token.copy_and_replace("typing_key", typing_key) - typing_by_room = {event["room_id"]: event for event in typing} + typing_by_room = {event["room_id"]: [event] for event in typing} + for event in typing: + event.pop("room_id") logger.debug("Typing %r", typing_by_room) rm_handler = self.hs.get_handlers().room_member_handler @@ -256,7 +258,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch, state=state, limited=False, - typing=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, []) ) if room_sync: rooms.append(room_sync) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 76489e27c8..2ae2eec55d 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -66,6 +66,7 @@ class SyncRestServlet(RestServlet): } "state": [] // list of EventIDs updating the current state to // be what it should be at the end of the batch. + "ephemeral": [] }] } """ @@ -188,9 +189,8 @@ class SyncRestServlet(RestServlet): "state": state_event_ids, "limited": room.limited, "published": room.published, + "ephemeral": room.ephemeral, } - if room.typing is not None: - result["typing"] = room.typing return result -- cgit 1.4.1 From 4ad45f258245344218c5ac9ffcf20052bfdb5ba2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:41:49 +0000 Subject: Fix indent --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3860c3c95b..8c7681a48d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -158,7 +158,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def initial_sync_for_room(self, room_id, sync_config, now_token, - published_room_ids): + published_room_ids): """Sync a room for a client which is starting without any state Returns: A Deferred RoomSyncResult. -- cgit 1.4.1 From 78015948a7febb18e000651f72f8f58830a55b93 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 16:50:23 +0000 Subject: Initial implementation of auth conflict resolution --- synapse/events/utils.py | 6 +- synapse/federation/federation_client.py | 2 +- synapse/federation/federation_server.py | 33 +++++ synapse/federation/transport/client.py | 16 +++ synapse/federation/transport/server.py | 21 +++- synapse/handlers/federation.py | 207 ++++++++++++++++++++------------ synapse/storage/rejections.py | 4 +- tests/handlers/test_federation.py | 2 + 8 files changed, 210 insertions(+), 81 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index bcb5457278..10a6b9f264 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -45,12 +45,14 @@ def prune_event(event): "membership", ] + event_dict = event.get_dict() + new_content = {} def add_fields(*fields): for field in fields: if field in event.content: - new_content[field] = event.content[field] + new_content[field] = event_dict["content"][field] if event_type == EventTypes.Member: add_fields("membership") @@ -75,7 +77,7 @@ def prune_event(event): allowed_fields = { k: v - for k, v in event.get_dict().items() + for k, v in event_dict.items() if k in allowed_keys } diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index ebcd593506..1173ca817b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -345,7 +345,7 @@ class FederationClient(object): "auth_chain": [e.get_pdu_json(time_now) for e in local_auth], } - code, content = yield self.transport_layer.send_invite( + code, content = yield self.transport_layer.send_query_auth( destination=destination, room_id=room_id, event_id=event_id, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index fc5342afaa..8cff4e6472 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -230,6 +230,39 @@ class FederationServer(object): "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus], })) + @defer.inlineCallbacks + def on_query_auth_request(self, origin, content, event_id): + auth_chain = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content["auth_chain"] + ] + + missing = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content.get("missing", []) + ] + + ret = yield self.handler.on_query_auth( + origin, event_id, auth_chain, content.get("rejects", []), missing + ) + + time_now = self._clock.time_msec() + send_content = { + "auth_chain": [ + e.get_pdu_json(time_now) + for e in ret["auth_chain"] + ], + "rejects": content.get("rejects", []), + "missing": [ + e.get_pdu_json(time_now) + for e in ret.get("missing", []) + ], + } + + defer.returnValue( + (200, send_content) + ) + @log_function def _get_persisted_pdu(self, origin, event_id, do_auth=True): """ Get a PDU from the database with given origin and id. diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index e634a3a213..4cb1dea2de 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -213,3 +213,19 @@ class TransportLayerClient(object): ) defer.returnValue(response) + + @defer.inlineCallbacks + @log_function + def send_query_auth(self, destination, room_id, event_id, content): + path = PREFIX + "/query_auth/%s/%s" % (room_id, event_id) + + code, content = yield self.client.post_json( + destination=destination, + path=path, + data=content, + ) + + if not 200 <= code < 300: + raise RuntimeError("Got %d from send_invite", code) + + defer.returnValue(json.loads(content)) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index a380a6910b..9c9f8d525b 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -42,7 +42,7 @@ class TransportLayerServer(object): content = None origin = None - if request.method == "PUT": + if request.method in ["PUT", "POST"]: # TODO: Handle other method types? other content types? try: content_bytes = request.content.read() @@ -234,6 +234,16 @@ class TransportLayerServer(object): ) ) ) + self.server.register_path( + "POST", + re.compile("^" + PREFIX + "/query_auth/([^/]*)/([^/]*)$"), + self._with_authentication( + lambda origin, content, query, context, event_id: + self._on_query_auth_request( + origin, content, event_id, + ) + ) + ) @defer.inlineCallbacks @log_function @@ -325,3 +335,12 @@ class TransportLayerServer(object): ) defer.returnValue((200, content)) + + @defer.inlineCallbacks + @log_function + def _on_query_auth_request(self, origin, content, event_id): + new_content = yield self.request_handler.on_query_auth_request( + origin, content, event_id + ) + + defer.returnValue((200, new_content)) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 97e3c503b9..14c26d8cea 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -126,7 +126,7 @@ class FederationHandler(BaseHandler): if not state: state, auth_chain = yield replication.get_state_for_room( - origin, context=event.room_id, event_id=event.event_id, + origin, room_id=event.room_id, event_id=event.event_id, ) if not auth_chain: @@ -139,7 +139,7 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e, fetch_auth_from=origin) + yield self._handle_new_event(origin, e) except: logger.exception( "Failed to handle auth event %s", @@ -152,7 +152,7 @@ class FederationHandler(BaseHandler): for e in state: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e) + yield self._handle_new_event(origin, e) except: logger.exception( "Failed to handle state event %s", @@ -161,6 +161,7 @@ class FederationHandler(BaseHandler): try: yield self._handle_new_event( + origin, event, state=state, backfilled=backfilled, @@ -363,7 +364,14 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e) + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + target_host, e, auth_events=auth + ) except: logger.exception( "Failed to handle auth event %s", @@ -374,8 +382,13 @@ class FederationHandler(BaseHandler): # FIXME: Auth these. e.internal_metadata.outlier = True try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } yield self._handle_new_event( - e, fetch_auth_from=target_host + target_host, e, auth_events=auth ) except: logger.exception( @@ -384,6 +397,7 @@ class FederationHandler(BaseHandler): ) yield self._handle_new_event( + target_host, new_event, state=state, current_state=state, @@ -450,7 +464,7 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False - context = yield self._handle_new_event(event) + context = yield self._handle_new_event(origin, event) logger.debug( "on_send_join_request: After _handle_new_event: %s, sigs: %s", @@ -651,11 +665,12 @@ class FederationHandler(BaseHandler): waiters.pop().callback(None) @defer.inlineCallbacks - def _handle_new_event(self, event, state=None, backfilled=False, - current_state=None, fetch_auth_from=None): + @log_function + def _handle_new_event(self, origin, event, state=None, backfilled=False, + current_state=None, auth_events=None): logger.debug( - "_handle_new_event: Before annotate: %s, sigs: %s", + "_handle_new_event: %s, sigs: %s", event.event_id, event.signatures, ) @@ -663,62 +678,34 @@ class FederationHandler(BaseHandler): event, old_state=state ) + if not auth_events: + auth_events = context.auth_events + logger.debug( - "_handle_new_event: Before auth fetch: %s, sigs: %s", - event.event_id, event.signatures, + "_handle_new_event: %s, auth_events: %s", + event.event_id, auth_events, ) is_new_state = not event.internal_metadata.is_outlier() - known_ids = set( - [s.event_id for s in context.auth_events.values()] - ) - - for e_id, _ in event.auth_events: - if e_id not in known_ids: - e = yield self.store.get_event(e_id, allow_none=True) - - if not e and fetch_auth_from is not None: - # Grab the auth_chain over federation if we are missing - # auth events. - auth_chain = yield self.replication_layer.get_event_auth( - fetch_auth_from, event.event_id, event.room_id - ) - for auth_event in auth_chain: - yield self._handle_new_event(auth_event) - e = yield self.store.get_event(e_id, allow_none=True) - - if not e: - # TODO: Do some conflict res to make sure that we're - # not the ones who are wrong. - logger.info( - "Rejecting %s as %s not in db or %s", - event.event_id, e_id, known_ids, - ) - # FIXME: How does raising AuthError work with federation? - raise AuthError(403, "Cannot find auth event") - - context.auth_events[(e.type, e.state_key)] = e - - logger.debug( - "_handle_new_event: Before hack: %s, sigs: %s", - event.event_id, event.signatures, - ) - + # This is a hack to fix some old rooms where the initial join event + # didn't reference the create event in its auth events. if event.type == EventTypes.Member and not event.auth_events: if len(event.prev_events) == 1: c = yield self.store.get_event(event.prev_events[0][0]) if c.type == EventTypes.Create: - context.auth_events[(c.type, c.state_key)] = c - - logger.debug( - "_handle_new_event: Before auth check: %s, sigs: %s", - event.event_id, event.signatures, - ) + auth_events[(c.type, c.state_key)] = c try: - self.auth.check(event, auth_events=context.auth_events) - except AuthError: + yield self.do_auth( + origin, event, context, auth_events=auth_events + ) + except AuthError as e: + logger.warn( + "Rejecting %s because %s", + event.event_id, e.msg + ) + # TODO: Store rejection. context.rejected = RejectedReason.AUTH_ERROR @@ -731,11 +718,6 @@ class FederationHandler(BaseHandler): ) raise - logger.debug( - "_handle_new_event: Before persist_event: %s, sigs: %s", - event.event_id, event.signatures, - ) - yield self.store.persist_event( event, context=context, @@ -744,25 +726,73 @@ class FederationHandler(BaseHandler): current_state=current_state, ) - logger.debug( - "_handle_new_event: After persist_event: %s, sigs: %s", - event.event_id, event.signatures, + defer.returnValue(context) + + @defer.inlineCallbacks + def on_query_auth(self, origin, event_id, remote_auth_chain, rejects, + missing): + # Just go through and process each event in `remote_auth_chain`. We + # don't want to fall into the trap of `missing` being wrong. + for e in remote_auth_chain: + try: + yield self._handle_new_event(origin, e) + except AuthError: + pass + + # Now get the current auth_chain for the event. + local_auth_chain = yield self.store.get_auth_chain([event_id]) + + # TODO: Check if we would now reject event_id. If so we need to tell + # everyone. + + ret = yield self.construct_auth_difference( + local_auth_chain, remote_auth_chain ) - defer.returnValue(context) + logger.debug("on_query_auth reutrning: %s", ret) + + defer.returnValue(ret) @defer.inlineCallbacks - def do_auth(self, origin, event, context): - for e_id, _ in event.auth_events: - pass + @log_function + def do_auth(self, origin, event, context, auth_events): + # Check if we have all the auth events. + res = yield self.store.have_events( + [e_id for e_id, _ in event.auth_events] + ) - auth_events = set(e_id for e_id, _ in event.auth_events) - current_state = set(e.event_id for e in context.auth_events.values()) + event_auth_events = set(e_id for e_id, _ in event.auth_events) + seen_events = set(res.keys()) - missing_auth = auth_events - current_state + missing_auth = event_auth_events - seen_events if missing_auth: + logger.debug("Missing auth: %s", missing_auth) + # If we don't have all the auth events, we need to get them. + remote_auth_chain = yield self.replication_layer.get_event_auth( + origin, event.room_id, event.event_id + ) + + for e in remote_auth_chain: + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in remote_auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass + + current_state = set(e.event_id for e in auth_events.values()) + different_auth = event_auth_events - current_state + + if different_auth and not event.internal_metadata.is_outlier(): # Do auth conflict res. + logger.debug("Different auth: %s", different_auth) # 1. Get what we think is the auth chain. auth_ids = self.auth.compute_auth_events(event, context) @@ -778,14 +808,24 @@ class FederationHandler(BaseHandler): # 3. Process any remote auth chain events we haven't seen. for e in result.get("missing", []): - # TODO. - pass + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in result["auth_chain"] + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass # 4. Look at rejects and their proofs. # TODO. try: - self.auth.check(event, auth_events=context.auth_events) + self.auth.check(event, auth_events=auth_events) except AuthError: raise @@ -802,12 +842,16 @@ class FederationHandler(BaseHandler): dict """ + logger.debug("construct_auth_difference Start!") + # TODO: Make sure we are OK with local_auth or remote_auth having more # auth events in them than strictly necessary. def sort_fun(ev): return ev.depth, ev.event_id + logger.debug("construct_auth_difference after sort_fun!") + # We find the differences by starting at the "bottom" of each list # and iterating up on both lists. The lists are ordered by depth and # then event_id, we iterate up both lists until we find the event ids @@ -823,11 +867,18 @@ class FederationHandler(BaseHandler): local_iter = iter(local_list) remote_iter = iter(remote_list) - current_local = local_iter.next() - current_remote = remote_iter.next() + logger.debug("construct_auth_difference before get_next!") def get_next(it, opt=None): - return it.next() if it.has_next() else opt + try: + return it.next() + except: + return opt + + current_local = get_next(local_iter) + current_remote = get_next(remote_iter) + + logger.debug("construct_auth_difference before while") missing_remotes = [] missing_locals = [] @@ -867,6 +918,8 @@ class FederationHandler(BaseHandler): current_remote = get_next(remote_iter) continue + logger.debug("construct_auth_difference after while") + # missing locals should be sent to the server # We should find why we are missing remotes, as they will have been # rejected. @@ -886,6 +939,7 @@ class FederationHandler(BaseHandler): reason = yield self.store.get_rejection_reason(e.event_id) if reason is None: # FIXME: ERRR?! + logger.warn("Could not find reason for %s", e.event_id) raise RuntimeError("") reason_map[e.event_id] = reason @@ -899,7 +953,10 @@ class FederationHandler(BaseHandler): # TODO: Get proof. pass + logger.debug("construct_auth_difference returning") + defer.returnValue({ + "auth_chain": local_auth, "rejects": { e.event_id: { "reason": reason_map[e.event_id], diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py index b7249700d7..4e1a9a2783 100644 --- a/synapse/storage/rejections.py +++ b/synapse/storage/rejections.py @@ -28,12 +28,12 @@ class RejectionsStore(SQLBaseStore): values={ "event_id": event_id, "reason": reason, - "last_failure": self._clock.time_msec(), + "last_check": self._clock.time_msec(), } ) def get_rejection_reason(self, event_id): - self._simple_select_one_onecol( + return self._simple_select_one_onecol( table="rejections", retcol="reason", keyvalues={ diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index ed21defd13..44dbce6bea 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -52,6 +52,7 @@ class FederationTestCase(unittest.TestCase): "get_room", "get_destination_retry_timings", "set_destination_retry_timings", + "have_events", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -90,6 +91,7 @@ class FederationTestCase(unittest.TestCase): self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) + self.datastore.have_events.return_value = defer.succeed({}) def annotate(ev, old_state=None): context = Mock() -- cgit 1.4.1 From ece828a7b72a25502d8daa15f6429c73a620b53c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 18:15:24 +0000 Subject: Update todo for the filtering on sync --- synapse/handlers/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 8c7681a48d..5768702192 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -290,7 +290,8 @@ class SyncHandler(BaseHandler): """ # TODO(mjark): Check if they have joined the room between # the previous sync and this one. - # TODO(mjark): Apply the event filter in sync_config + # TODO(mjark): Apply the event filter in sync_config taking care to get + # enough events to reach the limit # TODO(mjark): Check for redactions we might have missed. recents, token = yield self.store.get_recent_events_for_room( room_id, -- cgit 1.4.1 From c1d860870b418b9583078022d0babe557b73760f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 10:48:47 +0000 Subject: Fix regression where we no longer correctly handled the case of gaps in our event graph --- synapse/federation/federation_server.py | 3 +++ synapse/handlers/federation.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 8cff4e6472..845a07a3a3 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -366,10 +366,13 @@ class FederationServer(object): logger.debug("Processed pdu %s", event_id) else: logger.warn("Failed to get PDU %s", event_id) + fetch_state = True except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") fetch_state = True + else: + fetch_state = True else: fetch_state = True diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 14c26d8cea..de47a97e6b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -119,7 +119,7 @@ class FederationHandler(BaseHandler): event.room_id, self.server_name ) - if not is_in_room and not event.internal_metadata.outlier: + if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") replication = self.replication_layer @@ -780,6 +780,7 @@ class FederationHandler(BaseHandler): (e.type, e.state_key): e for e in remote_auth_chain if e.event_id in auth_ids } + e.internal_metadata.outlier = True yield self._handle_new_event( origin, e, auth_events=auth ) @@ -787,6 +788,8 @@ class FederationHandler(BaseHandler): except AuthError: pass + # FIXME: Assumes we have and stored all the state for all the + # prev_events current_state = set(e.event_id for e in auth_events.values()) different_auth = event_auth_events - current_state @@ -814,6 +817,7 @@ class FederationHandler(BaseHandler): (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } + e.internal_metadata.outlier = True yield self._handle_new_event( origin, e, auth_events=auth ) @@ -882,7 +886,7 @@ class FederationHandler(BaseHandler): missing_remotes = [] missing_locals = [] - while current_local and current_remote: + while current_local or current_remote: if current_remote is None: missing_locals.append(current_local) current_local = get_next(local_iter) -- cgit 1.4.1 From 0c2d245fdf5f24f99138f1c60c4e6bed52cf5d5a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:08:52 +0000 Subject: Update the current state of an event if we update auth events. --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index de47a97e6b..cc22f21cd1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -706,7 +706,6 @@ class FederationHandler(BaseHandler): event.event_id, e.msg ) - # TODO: Store rejection. context.rejected = RejectedReason.AUTH_ERROR yield self.store.persist_event( @@ -828,6 +827,9 @@ class FederationHandler(BaseHandler): # 4. Look at rejects and their proofs. # TODO. + context.current_state.update(auth_events) + context.state_group = None + try: self.auth.check(event, auth_events=auth_events) except AuthError: -- cgit 1.4.1 From 22dd1cde2d83a2448074816108b85d1957315236 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:32:35 +0000 Subject: Filter the recent events before applying the limit when doing an incremental sync with a gap --- synapse/api/filtering.py | 2 -- synapse/handlers/sync.py | 53 ++++++++++++++++++++++++++---------- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/storage/stream.py | 21 ++++++++++---- 4 files changed, 54 insertions(+), 24 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index b7e5d3222f..fa4de2614d 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5768702192..0df1851b0e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -278,6 +278,40 @@ class SyncHandler(BaseHandler): next_batch=now_token, )) + @defer.inlineCallbacks + def load_filtered_recents(self, room_id, sync_config, since_token, + now_token): + limited = True + recents = [] + filtering_factor = 2 + load_limit = max(sync_config.limit * filtering_factor, 100) + max_repeat = 3 # Only try a few times per room, otherwise + room_key = now_token.room_key + + while limited and len(recents) < sync_config.limit and max_repeat: + events, room_key = yield self.store.get_recent_events_for_room( + room_id, + limit=load_limit + 1, + from_token=since_token.room_key, + end_token=room_key, + ) + loaded_recents = sync_config.filter.filter_room_events(events) + loaded_recents.extend(recents) + recents = loaded_recents + if len(events) <= load_limit: + limited = False + max_repeat -= 1 + + if len(recents) > sync_config.limit: + recents = recents[-sync_config.limit:] + room_key = recents[0].internal_metadata.before + + prev_batch_token = now_token.copy_and_replace( + "room_key", room_key + ) + + defer.returnValue((recents, prev_batch_token, limited)) + @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, @@ -288,28 +322,17 @@ class SyncHandler(BaseHandler): Returns: A Deferred RoomSyncResult """ + # TODO(mjark): Check if they have joined the room between # the previous sync and this one. - # TODO(mjark): Apply the event filter in sync_config taking care to get - # enough events to reach the limit # TODO(mjark): Check for redactions we might have missed. - recents, token = yield self.store.get_recent_events_for_room( - room_id, - limit=sync_config.limit + 1, - from_token=since_token.room_key, - end_token=now_token.room_key, + + recents, prev_batch_token, limited = self.load_filtered_recents( + room_id, sync_config, since_token, ) logging.debug("Recents %r", recents) - if len(recents) > sync_config.limit: - limited = True - recents = recents[1:] - else: - limited = False - - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - # TODO(mjark): This seems racy since this isn't being passed a # token to indicate what point in the stream this is current_state_events = yield self.state_handler.get_current_state( diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index c1277d2675..46ea50d118 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -116,7 +116,7 @@ class SyncRestServlet(RestServlet): user.localpart, filter_id ) except: - filter = Filter({}) + filter = Filter({}) # filter = filter.apply_overrides(http_request) #if filter.matches(event): # # stuff diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 2ea5e1a021..73504c8b52 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -181,15 +181,11 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) - for event, row in zip(ret, rows): - stream = row["stream_ordering"] - topo = event.depth - internal = event.internal_metadata - internal.before = str(_StreamToken(topo, stream - 1)) - internal.after = str(_StreamToken(topo, stream)) + self._set_before_and_after(ret, rows) if rows: key = "s%d" % max([r["stream_ordering"] for r in rows]) + else: # Assume we didn't get anything because there was nothing to # get. @@ -267,6 +263,8 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + self._set_before_and_after(events, rows) + return events, next_token, return self.runInteraction("paginate_room_events", f) @@ -328,6 +326,8 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + self._set_before_and_after(events, rows) + return events, token return self.runInteraction( @@ -354,3 +354,12 @@ class StreamStore(SQLBaseStore): key = res[0]["m"] return "s%d" % (key,) + + @staticmethod + def _set_before_and_after(events, rows): + for event, row in zip(events, rows): + stream = row["stream_ordering"] + topo = event.depth + internal = event.internal_metadata + internal.before = str(_StreamToken(topo, stream - 1)) + internal.after = str(_StreamToken(topo, stream)) -- cgit 1.4.1 From e97de6d96a25912b34fa38cd3a587d798f6b676c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:35:20 +0000 Subject: Filter the recent events before applying the limit when doing an initial sync --- synapse/handlers/sync.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 0df1851b0e..b83fcad655 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -163,12 +163,11 @@ class SyncHandler(BaseHandler): Returns: A Deferred RoomSyncResult. """ - recent_events, token = yield self.store.get_recent_events_for_room( - room_id, - limit=sync_config.limit, - end_token=now_token.room_key, + + recents, prev_batch_token, limited = self.load_filtered_recents( + room_id, sync_config, now_token, ) - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state_events = yield self.state_handler.get_current_state( room_id ) @@ -176,10 +175,10 @@ class SyncHandler(BaseHandler): defer.returnValue(RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, - events=recent_events, + events=recents, prev_batch=prev_batch_token, state=current_state_events, - limited=True, + limited=limited, ephemeral=[], )) @@ -279,8 +278,8 @@ class SyncHandler(BaseHandler): )) @defer.inlineCallbacks - def load_filtered_recents(self, room_id, sync_config, since_token, - now_token): + def load_filtered_recents(self, room_id, sync_config, now_token, + since_token=None): limited = True recents = [] filtering_factor = 2 @@ -292,7 +291,7 @@ class SyncHandler(BaseHandler): events, room_key = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, - from_token=since_token.room_key, + from_token=since_token.room_key if since_token else None, end_token=room_key, ) loaded_recents = sync_config.filter.filter_room_events(events) @@ -328,7 +327,7 @@ class SyncHandler(BaseHandler): # TODO(mjark): Check for redactions we might have missed. recents, prev_batch_token, limited = self.load_filtered_recents( - room_id, sync_config, since_token, + room_id, sync_config, now_token, since_token, ) logging.debug("Recents %r", recents) -- cgit 1.4.1 From 8498d348d818aa2d2cb9bb9bb2775103840f355d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:42:09 +0000 Subject: Fix token formatting --- synapse/handlers/sync.py | 6 +++--- synapse/storage/stream.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b83fcad655..3c68e2a9ec 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -164,7 +164,7 @@ class SyncHandler(BaseHandler): A Deferred RoomSyncResult. """ - recents, prev_batch_token, limited = self.load_filtered_recents( + recents, prev_batch_token, limited = yield self.load_filtered_recents( room_id, sync_config, now_token, ) @@ -288,7 +288,7 @@ class SyncHandler(BaseHandler): room_key = now_token.room_key while limited and len(recents) < sync_config.limit and max_repeat: - events, room_key = yield self.store.get_recent_events_for_room( + events, (room_key,_) = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, @@ -326,7 +326,7 @@ class SyncHandler(BaseHandler): # the previous sync and this one. # TODO(mjark): Check for redactions we might have missed. - recents, prev_batch_token, limited = self.load_filtered_recents( + recents, prev_batch_token, limited = yield self.load_filtered_recents( room_id, sync_config, now_token, since_token, ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 73504c8b52..3ccb6f8a61 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -316,9 +316,9 @@ class StreamStore(SQLBaseStore): toke = rows[0]["stream_ordering"] - 1 start_token = str(_StreamToken(topo, toke)) - token = (start_token, end_token) + token = (start_token, str(end_token)) else: - token = (end_token, end_token) + token = (str(end_token), str(end_token)) events = self._get_events_txn( txn, -- cgit 1.4.1 From 4a67834bc84f604605c618049599f4638434c7cf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:50:15 +0000 Subject: Pass client info to the sync_config --- synapse/handlers/sync.py | 5 +++-- synapse/rest/client/v2_alpha/sync.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3c68e2a9ec..1a74a4c97c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", - "device", + "client_info", "limit", "gap", "sort", @@ -288,12 +288,13 @@ class SyncHandler(BaseHandler): room_key = now_token.room_key while limited and len(recents) < sync_config.limit and max_repeat: - events, (room_key,_) = yield self.store.get_recent_events_for_room( + events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, end_token=room_key, ) + (room_key, _) = keys loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 46ea50d118..81d5cf8ead 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -123,7 +123,7 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, - device="TODO", # TODO(mjark) Get the device_id from access_token + client_info=client, gap=gap, limit=limit, sort=sort, -- cgit 1.4.1 From a70a801184814d116ed5b10a952e17c45df7bfc8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 13:34:01 +0000 Subject: Fix bug where we superfluously asked for current state. Change API of /query_auth/ so that we don't duplicate events in the response. --- synapse/api/auth.py | 2 ++ synapse/federation/federation_client.py | 7 +---- synapse/federation/federation_server.py | 12 ++++---- synapse/handlers/federation.py | 51 ++++++++++++--------------------- synapse/state.py | 20 ++++++++++--- 5 files changed, 43 insertions(+), 49 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3471afd7e7..37e31d2b6f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -102,6 +102,8 @@ class Auth(object): def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) + logger.debug("Got curr_state %s", curr_state) + for event in curr_state: if event.type == EventTypes.Member: try: diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 1173ca817b..e1539bd0e0 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -357,15 +357,10 @@ class FederationClient(object): for e in content["auth_chain"] ] - missing = [ - (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) - for e in content.get("missing", []) - ] - ret = { "auth_chain": auth_chain, "rejects": content.get("rejects", []), - "missing": missing, + "missing": content.get("missing", []), } defer.returnValue(ret) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 845a07a3a3..84ed0a0ba0 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -252,11 +252,8 @@ class FederationServer(object): e.get_pdu_json(time_now) for e in ret["auth_chain"] ], - "rejects": content.get("rejects", []), - "missing": [ - e.get_pdu_json(time_now) - for e in ret.get("missing", []) - ], + "rejects": ret.get("rejects", []), + "missing": ret.get("missing", []), } defer.returnValue( @@ -372,7 +369,10 @@ class FederationServer(object): logger.exception("Failed to get PDU") fetch_state = True else: - fetch_state = True + prevs = {e_id for e_id, _ in pdu.prev_events} + seen = set(have_seen.keys()) + if prevs - seen: + fetch_state = True else: fetch_state = True diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index cc22f21cd1..35cad4182a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -121,38 +121,18 @@ class FederationHandler(BaseHandler): ) if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") - - replication = self.replication_layer - - if not state: - state, auth_chain = yield replication.get_state_for_room( - origin, room_id=event.room_id, event_id=event.event_id, - ) - - if not auth_chain: - auth_chain = yield replication.get_event_auth( - origin, - context=event.room_id, - event_id=event.event_id, - ) - - for e in auth_chain: - e.internal_metadata.outlier = True - try: - yield self._handle_new_event(origin, e) - except: - logger.exception( - "Failed to handle auth event %s", - e.event_id, - ) - current_state = state - if state: + if state and auth_chain is not None: for e in state: e.internal_metadata.outlier = True try: - yield self._handle_new_event(origin, e) + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event(origin, e, auth_events=auth) except: logger.exception( "Failed to handle state event %s", @@ -809,18 +789,23 @@ class FederationHandler(BaseHandler): ) # 3. Process any remote auth chain events we haven't seen. - for e in result.get("missing", []): + for missing_id in result.get("missing", []): try: - auth_ids = [e_id for e_id, _ in e.auth_events] + for e in result["auth_chain"]: + if e.event_id == missing_id: + ev = e + break + + auth_ids = [e_id for e_id, _ in ev.auth_events] auth = { (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } - e.internal_metadata.outlier = True + ev.internal_metadata.outlier = True yield self._handle_new_event( - origin, e, auth_events=auth + origin, ev, auth_events=auth ) - auth_events[(e.type, e.state_key)] = e + auth_events[(ev.type, ev.state_key)] = ev except AuthError: pass @@ -970,5 +955,5 @@ class FederationHandler(BaseHandler): } for e in base_remote_rejected }, - "missing": missing_locals, + "missing": [e.event_id for e in missing_locals], }) diff --git a/synapse/state.py b/synapse/state.py index d9fdfb34be..e6632978b5 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -166,10 +166,17 @@ class StateHandler(object): first is the name of a state group if one and only one is involved, otherwise `None`. """ + logger.debug("resolve_state_groups event_ids %s", event_ids) + state_groups = yield self.store.get_state_groups( event_ids ) + logger.debug( + "resolve_state_groups state_groups %s", + state_groups.keys() + ) + group_names = set(state_groups.keys()) if len(group_names) == 1: name, state_list = state_groups.items().pop() @@ -205,6 +212,15 @@ class StateHandler(object): if len(v.values()) > 1 } + logger.debug( + "resolve_state_groups Unconflicted state: %s", + unconflicted_state.values(), + ) + logger.debug( + "resolve_state_groups Conflicted state: %s", + conflicted_state.values(), + ) + if event_type: prev_states_events = conflicted_state.get( (event_type, state_key), [] @@ -240,10 +256,6 @@ class StateHandler(object): 1. power levels 2. memberships 3. other events. - - :param conflicted_state: - :param auth_events: - :return: """ resolved_state = {} power_key = (EventTypes.PowerLevels, "") -- cgit 1.4.1 From 8fe39a03119b3d092e84319d6d99b6ffc49a83f5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 13:33:41 +0000 Subject: Check if the user has joined the room between incremental syncs --- synapse/handlers/sync.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1a74a4c97c..9e07f30b24 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -16,7 +16,7 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig -from synapse.api.constants import Membership +from synapse.api.constants import Membership, EventTypes from twisted.internet import defer @@ -250,6 +250,11 @@ class SyncHandler(BaseHandler): ) else: prev_batch = now_token + + state = yield self.check_joined_room( + sync_config, room_id, state + ) + room_sync = RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, @@ -323,8 +328,6 @@ class SyncHandler(BaseHandler): A Deferred RoomSyncResult """ - # TODO(mjark): Check if they have joined the room between - # the previous sync and this one. # TODO(mjark): Check for redactions we might have missed. recents, prev_batch_token, limited = yield self.load_filtered_recents( @@ -349,6 +352,10 @@ class SyncHandler(BaseHandler): current_state=current_state_events, ) + state_events_delta = yield self.check_joined_room( + sync_config, room_id, state_events_delta + ) + room_sync = RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, @@ -356,7 +363,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, - typing=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, None) ) logging.debug("Room sync: %r", room_sync) @@ -402,3 +409,19 @@ class SyncHandler(BaseHandler): if event.event_id not in previous_dict: state_delta.append(event) return state_delta + + @defer.inlineCallbacks + def check_joined_room(self, sync_config, room_id, state_delta): + joined = False + for event in state_delta: + if ( + event.type == EventTypes.Member + and event.state_key == sync_config.user.to_string() + ): + if event.content["membership"] == Membership.JOIN: + joined = True + + if joined: + state_delta = yield self.state_handler.get_current_state(room_id) + + defer.returnValue(state_delta) -- cgit 1.4.1 From 017dfaef4c0d4550acd45d27cdcea1c20766684d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 15:52:05 +0000 Subject: Add doc string for __nonzero__ overrides for sync results, raise not implemented if the client attempts to do a gapless sync --- synapse/handlers/sync.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9e07f30b24..dc69b3cfe7 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -49,6 +49,9 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ __slots__ = [] def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if room needs to be part of the sync result. + """ return bool(self.events or self.state or self.ephemeral) @@ -61,6 +64,10 @@ class SyncResult(collections.namedtuple("SyncResult", [ __slots__ = [] def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if the notifier needs to wait for more events when polling for + events. + """ return bool( self.private_user_data or self.public_user_data or self.rooms ) @@ -108,7 +115,7 @@ class SyncHandler(BaseHandler): return self.incremental_sync_with_gap(sync_config, since_token) else: #TODO(mjark): Handle gapless sync - pass + raise NotImplementedError() @defer.inlineCallbacks def initial_sync(self, sync_config): -- cgit 1.4.1 From b724a809c46f782231e18b70a632ce1ea9c540da Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 15:57:53 +0000 Subject: Only auth_events with event if event in event.auth_events --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 35cad4182a..40836e0c51 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -805,7 +805,9 @@ class FederationHandler(BaseHandler): yield self._handle_new_event( origin, ev, auth_events=auth ) - auth_events[(ev.type, ev.state_key)] = ev + + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev except AuthError: pass -- cgit 1.4.1 From 2cd29dbdd91a8f58f988eb3c417ad6a187fc7e87 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 16:51:58 +0000 Subject: Fix bug where accepting invite over federation didn't work. Add logging. --- synapse/handlers/federation.py | 57 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 8 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 40836e0c51..623c54d584 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -343,6 +343,10 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True + + if e.event_id == event.event_id: + continue + try: auth_ids = [e_id for e_id, _ in e.auth_events] auth = { @@ -359,7 +363,9 @@ class FederationHandler(BaseHandler): ) for e in state: - # FIXME: Auth these. + if e.event_id == event.event_id: + continue + e.internal_metadata.outlier = True try: auth_ids = [e_id for e_id, _ in e.auth_events] @@ -376,11 +382,18 @@ class FederationHandler(BaseHandler): e.event_id, ) + auth_ids = [e_id for e_id, _ in event.auth_events] + auth_events = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( target_host, new_event, state=state, current_state=state, + auth_events=auth_events, ) yield self.notifier.on_new_room_event( @@ -752,7 +765,17 @@ class FederationHandler(BaseHandler): origin, event.room_id, event.event_id ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in remote_auth_chain] + ) + for e in remote_auth_chain: + if e.event_id in seen_remotes.keys(): + continue + + if e.event_id == event.event_id: + continue + try: auth_ids = [e_id for e_id, _ in e.auth_events] auth = { @@ -760,10 +783,17 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } e.internal_metadata.outlier = True + + logger.debug( + "do_auth %s missing_auth: %s", + event.event_id, e.event_id + ) yield self._handle_new_event( origin, e, auth_events=auth ) - auth_events[(e.type, e.state_key)] = e + + if e.event_id in event_auth_events: + auth_events[(e.type, e.state_key)] = e except AuthError: pass @@ -788,20 +818,31 @@ class FederationHandler(BaseHandler): local_auth_chain, ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) + # 3. Process any remote auth chain events we haven't seen. - for missing_id in result.get("missing", []): - try: - for e in result["auth_chain"]: - if e.event_id == missing_id: - ev = e - break + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue + if ev.event_id == event.event_id: + continue + + try: auth_ids = [e_id for e_id, _ in ev.auth_events] auth = { (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } ev.internal_metadata.outlier = True + + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) + yield self._handle_new_event( origin, ev, auth_events=auth ) -- cgit 1.4.1 From 0b1cc7cc0bd81f93ed353a6ac67e06f91b62ab90 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 16:24:40 +0000 Subject: Return empty list rather than None when there are no emphemeral events for a room --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index dc69b3cfe7..962686f4bb 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -370,7 +370,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, - ephemeral=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, []) ) logging.debug("Room sync: %r", room_sync) -- cgit 1.4.1 From 4c0da49d7ca58774adad870da98a7e168a5be18c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 22:53:13 +0000 Subject: Resign events when we return them via /query_auth/ --- synapse/handlers/federation.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 623c54d584..8bf5a4cc11 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -741,6 +741,15 @@ class FederationHandler(BaseHandler): local_auth_chain, remote_auth_chain ) + for event in ret["auth_chain"]: + event.signatures.update( + compute_event_signature( + event, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) + logger.debug("on_query_auth reutrning: %s", ret) defer.returnValue(ret) -- cgit 1.4.1 From 1a2de0c5feb1183b35045bb7fb9e379a9598d1cb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Feb 2015 17:39:41 +0000 Subject: Implement txns for AS (un)registration. --- synapse/handlers/appservice.py | 3 +- synapse/storage/__init__.py | 2 +- synapse/storage/appservice.py | 120 +++++++++++++++++++++++++++++++++-------- 3 files changed, 102 insertions(+), 23 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8bd475cbfd..da994ba8e0 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -43,7 +43,8 @@ class ApplicationServicesHandler(BaseHandler): "Consult the home server admin.", errcode=Codes.FORBIDDEN ) - # TODO store this AS + logger.info("Updating application service info...") + yield self.store.update_app_service(app_service) def unregister(self, token): logger.info("Unregister as_token=%s", token) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1f207495f6..6ff0093136 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,7 +67,7 @@ SCHEMAS = [ "event_signatures", "pusher", "media_repository", - "application_services" + "application_services", "filtering", "rejections", ] diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 5a0e47e0d4..db0c546211 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -15,11 +15,17 @@ import logging from twisted.internet import defer +from synapse.api.errors import StoreError from ._base import SQLBaseStore logger = logging.getLogger(__name__) +namespace_enum = [ + "users", # 0 + "aliases", # 1 + "rooms" # 2 +] # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): @@ -30,25 +36,26 @@ class ApplicationService(object): def __init__(self, token, url=None, namespaces=None): self.token = token - if url: - self.url = url - if namespaces: - self._set_namespaces(namespaces) + self.url = url + self.namespaces = self._get_namespaces(namespaces) - def _set_namespaces(self, namespaces): + def _get_namespaces(self, namespaces): # Sanity check that it is of the form: # { # users: ["regex",...], # aliases: ["regex",...], # rooms: ["regex",...], # } + if not namespaces: + return None + for ns in ["users", "rooms", "aliases"]: if type(namespaces[ns]) != list: raise ValueError("Bad namespace value for '%s'", ns) for regex in namespaces[ns]: if not isinstance(regex, basestring): raise ValueError("Expected string regex for ns '%s'", ns) - self.namespaces = namespaces + return namespaces def is_interested(self, event): """Check if this service is interested in this event. @@ -110,10 +117,38 @@ class ApplicationServiceStore(SQLBaseStore): This removes all AS specific regex and the base URL. The token is the only thing preserved for future registration attempts. """ - # TODO: DELETE FROM application_services_regex WHERE id=this service - # TODO: SET url=NULL WHERE token=token - # TODO: Update cache - pass + yield self.runInteraction( + "unregister_app_service", + self._unregister_app_service_txn, + token, + ) + # update cache TODO: Should this be in the txn? + for service in self.cache.services: + if service.token == token: + service.url = None + service.namespaces = None + + def _unregister_app_service_txn(self, txn, token): + # kill the url to prevent pushes + txn.execute( + "UPDATE application_services SET url=NULL WHERE token=?", + (token,) + ) + + # cleanup regex + as_id = self._get_as_id_txn(txn, token) + if not as_id: + logger.warning( + "unregister_app_service_txn: Failed to find as_id for token=", + token + ) + return False + + txn.execute( + "DELETE FROM application_services_regex WHERE as_id=?", + (as_id,) + ) + return True def update_app_service(self, service): """Update an application service, clobbering what was previously there. @@ -124,12 +159,61 @@ class ApplicationServiceStore(SQLBaseStore): # NB: There is no "insert" since we provide no public-facing API to # allocate new ASes. It relies on the server admin inserting the AS # token into the database manually. + if not service.token or not service.url: + raise StoreError(400, "Token and url must be specified.") + + yield self.runInteraction( + "update_app_service", + self._update_app_service_txn, + service + ) + + # update cache TODO: Should this be in the txn? + for (index, cache_service) in enumerate(self.cache.services): + if service.token == cache_service.token: + self.cache.services[index] = service + logger.info("Updated: %s", service) + return + # new entry + self.cache.services.append(service) + logger.info("Updated(new): %s", service) + + def _update_app_service_txn(self, txn, service): + as_id = self._get_as_id_txn(txn, service.token) + if not as_id: + logger.warning( + "update_app_service_txn: Failed to find as_id for token=", + service.token + ) + return False + + txn.execute( + "UPDATE application_services SET url=? WHERE id=?", + (service.url, as_id,) + ) + # cleanup regex + txn.execute( + "DELETE FROM application_services_regex WHERE id=?", + (as_id,) + ) + for (ns_int, ns_str) in enumerate(namespace_enum): + if ns_str in service.namespaces: + for regex in service.namespaces[ns_str]: + txn.execute( + "INSERT INTO application_services_regex(" + "as_id, namespace, regex) values(?,?,?)", + (as_id, ns_int, regex) + ) + return True - # TODO: UPDATE application_services, SET url WHERE token=service.token - # TODO: DELETE FROM application_services_regex WHERE id=this service - # TODO: INSERT INTO application_services_regex - # TODO: Update cache - pass + def _get_as_id_txn(self, txn, token): + cursor = txn.execute( + "SELECT id FROM application_services WHERE token=?", + (token,) + ) + res = cursor.fetchone() + if res: + return res[0] def get_services_for_event(self, event): return self.cache.get_services_for_event(event) @@ -161,12 +245,6 @@ class ApplicationServiceStore(SQLBaseStore): sql = ("SELECT * FROM application_services LEFT JOIN " "application_services_regex ON application_services.id = " "application_services_regex.as_id") - - namespace_enum = [ - "users", # 0 - "aliases", # 1 - "rooms" # 2 - ] # SQL results in the form: # [ # { -- cgit 1.4.1 From e7ca813dd476c83497d4130ad8efa9424d86e921 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 10:38:14 +0000 Subject: Try to ensure we don't persist an event we have already persisted. In persist_event check if we already have the event, if so then update instead of replacing so that we don't cause a bump of the stream_ordering. --- synapse/handlers/federation.py | 42 ++++++++++++++++++++++++++------------- synapse/storage/__init__.py | 40 +++++++++++++++++++++++++++++++++---- tests/handlers/test_federation.py | 5 ++++- 3 files changed, 68 insertions(+), 19 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8bf5a4cc11..c384789c2f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -112,6 +112,14 @@ class FederationHandler(BaseHandler): logger.debug("Event: %s", event) + event_ids = set() + if state: + event_ids += {e.event_id for e in state} + if auth_chain: + event_ids += {e.event_id for e in auth_chain} + + seen_ids = (yield self.store.have_events(event_ids)).keys() + # FIXME (erikj): Awful hack to make the case where we are not currently # in the room work current_state = None @@ -124,20 +132,26 @@ class FederationHandler(BaseHandler): current_state = state if state and auth_chain is not None: - for e in state: - e.internal_metadata.outlier = True - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - yield self._handle_new_event(origin, e, auth_events=auth) - except: - logger.exception( - "Failed to handle state event %s", - e.event_id, - ) + for list_of_pdus in [auth_chain, state]: + for e in list_of_pdus: + if e.event_id in seen_ids: + continue + + e.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + except: + logger.exception( + "Failed to handle state event %s", + e.event_id, + ) try: yield self._handle_new_event( diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b4a7a3f068..93aefe0c48 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -161,6 +161,39 @@ class DataStore(RoomMemberStore, RoomStore, outlier = event.internal_metadata.is_outlier() + have_persisted = self._simple_select_one_onecol_txn( + txn, + table="event_json", + keyvalues={"event_id": event.event_id}, + retcol="event_id", + allow_none=True, + ) + + metadata_json = encode_canonical_json( + event.internal_metadata.get_dict() + ) + + if have_persisted: + if not outlier: + sql = ( + "UPDATE event_json SET internal_metadata = ?" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (metadata_json.decode("UTF-8"), event.event_id,) + ) + + sql = ( + "UPDATE events SET outlier = 0" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (event.event_id,) + ) + return + event_dict = { k: v for k, v in event.get_dict().items() @@ -170,10 +203,6 @@ class DataStore(RoomMemberStore, RoomStore, ] } - metadata_json = encode_canonical_json( - event.internal_metadata.get_dict() - ) - self._simple_insert_txn( txn, table="event_json", @@ -482,6 +511,9 @@ class DataStore(RoomMemberStore, RoomStore, the rejected reason string if we rejected the event, else maps to None. """ + if not event_ids: + return defer.succeed({}) + def f(txn): sql = ( "SELECT e.event_id, reason FROM events as e " diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 44dbce6bea..4270481139 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -91,7 +91,10 @@ class FederationTestCase(unittest.TestCase): self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) - self.datastore.have_events.return_value = defer.succeed({}) + + def have_events(event_ids): + return defer.succeed({}) + self.datastore.have_events.side_effect = have_events def annotate(ev, old_state=None): context = Mock() -- cgit 1.4.1 From 51969f9e5f4774e4d0ddcc2e8ebcf96803cbf295 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 10:40:14 +0000 Subject: Return rejected events if asked for it over federation. --- synapse/handlers/federation.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c384789c2f..0161fbe49c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -632,6 +632,7 @@ class FederationHandler(BaseHandler): event = yield self.store.get_event( event_id, allow_none=True, + allow_rejected=True, ) if event: -- cgit 1.4.1 From 4ff2273b300c0a40fcf054a613aac24c51af22f1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 11:23:26 +0000 Subject: Add FIXME note. --- synapse/handlers/federation.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0161fbe49c..322c1b407d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -716,6 +716,8 @@ class FederationHandler(BaseHandler): context.rejected = RejectedReason.AUTH_ERROR + # FIXME: Don't store as rejected with AUTH_ERROR if we haven't + # seen all the auth events. yield self.store.persist_event( event, context=context, -- cgit 1.4.1 From 06c34bfbaee553b851e0f02d8e17d5bff7360376 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 11:23:44 +0000 Subject: Give exception better message --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 322c1b407d..9d7557f578 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1000,7 +1000,7 @@ class FederationHandler(BaseHandler): if reason is None: # FIXME: ERRR?! logger.warn("Could not find reason for %s", e.event_id) - raise RuntimeError("") + raise RuntimeError("Could not find reason for %s" % e.event_id) reason_map[e.event_id] = reason -- cgit 1.4.1 From 197f3ea4bad066da251c7925336baab8bee296c9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 11:26:33 +0000 Subject: Implement regex checks for app services. Expose handler.get_services_for_event which manages the checks for all services. --- synapse/handlers/appservice.py | 25 +++++++++++++-- synapse/storage/appservice.py | 71 ++++++++++++++++++++++++++---------------- 2 files changed, 67 insertions(+), 29 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index da994ba8e0..9b8dd1bb49 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -34,9 +34,11 @@ class ApplicationServicesHandler(BaseHandler): logger.info("Register -> %s", app_service) # check the token is recognised try: - stored_service = yield self.store.get_app_service(app_service.token) + stored_service = yield self.store.get_app_service_by_token( + app_service.token + ) if not stored_service: - raise StoreError(404, "Application Service Not found") + raise StoreError(404, "Application service not found") except StoreError: raise SynapseError( 403, "Unrecognised application services token. " @@ -50,6 +52,25 @@ class ApplicationServicesHandler(BaseHandler): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) + def get_services_for_event(self, event): + """Retrieve a list of application services interested in this event. + + Args: + event(Event): The event to check. + Returns: + list: A list of services interested in this + event based on the service regex. + """ + # We need to know the aliases associated with this event.room_id, if any + alias_list = [] # TODO + + interested_list = [ + s for s in self.store.get_app_services() if ( + s.is_interested(event, alias_list) + ) + ] + return interested_list + def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index dd9b349370..c4e50be4c6 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import re from twisted.internet import defer +from synapse.api.constants import EventTypes from synapse.api.errors import StoreError from ._base import SQLBaseStore @@ -27,6 +29,7 @@ namespace_enum = [ "rooms" # 2 ] + # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): """Defines an application service. @@ -37,9 +40,9 @@ class ApplicationService(object): def __init__(self, token, url=None, namespaces=None): self.token = token self.url = url - self.namespaces = self._get_namespaces(namespaces) + self.namespaces = self._check_namespaces(namespaces) - def _get_namespaces(self, namespaces): + def _check_namespaces(self, namespaces): # Sanity check that it is of the form: # { # users: ["regex",...], @@ -57,22 +60,50 @@ class ApplicationService(object): raise ValueError("Expected string regex for ns '%s'", ns) return namespaces - def is_interested(self, event): + def _matches_regex(self, test_string, namespace_key): + for regex in self.namespaces[namespace_key]: + if re.match(regex, test_string): + return True + return False + + def _matches_user(self, event): + if (hasattr(event, "user_id") and + self._matches_regex(event.user_id, "users")): + return True + # also check m.room.member state key + if (hasattr(event, "type") and event.type == EventTypes.Member + and hasattr(event, "state_key") + and self._matches_regex(event.state_key, "users")): + return True + return False + + def _matches_room_id(self, event): + if hasattr(event, "room_id"): + return self._matches_regex(event.room_id, "rooms") + return False + + def _matches_aliases(self, event, alias_list): + for alias in alias_list: + if self._matches_regex(alias, "aliases"): + return True + return False + + def is_interested(self, event, aliases_for_event=None): """Check if this service is interested in this event. Args: event(Event): The event to check. + aliases_for_event(list): A list of all the known room aliases for + this event. Returns: bool: True if this service would like to know about this event. """ - # NB: This does not check room alias regex matches because that requires - # more context that an Event can provide. Room alias matches are checked - # in the ApplicationServiceHandler. - - # TODO check if event.room_id regex matches - # TODO check if event.user_id regex matches (or m.room.member state_key) + if aliases_for_event is None: + aliases_for_event = [] - return True + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) def __str__(self): return "ApplicationService: %s" % (self.__dict__,) @@ -89,20 +120,6 @@ class ApplicationServiceCache(object): def __init__(self): self.services = [] - def get_services_for_event(self, event): - """Retrieve a list of application services interested in this event. - - Args: - event(Event): The event to check. - Returns: - list: A list of services interested in this - event based on the service regex. - """ - interested_list = [ - s for s in self.services if s.is_event_claimed(event) - ] - return interested_list - class ApplicationServiceStore(SQLBaseStore): @@ -217,10 +234,10 @@ class ApplicationServiceStore(SQLBaseStore): if res: return res[0] - def get_services_for_event(self, event): - return self.cache.get_services_for_event(event) + def get_app_services(self): + return self.cache.services - def get_app_service(self, token, from_cache=True): + def get_app_service_by_token(self, token, from_cache=True): """Get the application service with the given token. Args: -- cgit 1.4.1 From a060b47b13037da56ed8db2978a297133c23fc7f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 13:17:28 +0000 Subject: Add namespace constants. Add restrict_to option to limit namespace checks. --- synapse/handlers/appservice.py | 25 +++++++++++++++---------- synapse/storage/appservice.py | 41 ++++++++++++++++++++++++----------------- 2 files changed, 39 insertions(+), 27 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 9b8dd1bb49..bf68b33398 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -52,11 +52,12 @@ class ApplicationServicesHandler(BaseHandler): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) - def get_services_for_event(self, event): + def get_services_for_event(self, event, restrict_to=""): """Retrieve a list of application services interested in this event. Args: event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. Returns: list: A list of services interested in this event based on the service regex. @@ -66,7 +67,7 @@ class ApplicationServicesHandler(BaseHandler): interested_list = [ s for s in self.store.get_app_services() if ( - s.is_interested(event, alias_list) + s.is_interested(event, restrict_to, alias_list) ) ] return interested_list @@ -80,11 +81,15 @@ class ApplicationServicesHandler(BaseHandler): Args: event(Event): The event to push out to interested services. """ - # TODO: Gather interested services - # get_services_for_event(event) <-- room IDs and user IDs - # Get a list of room aliases. Check regex. - # TODO: If unknown user: poke User Query API. - # TODO: If unknown room alias: poke Room Alias Query API. - - # TODO: Fork off pushes to these services - XXX First cut, best effort - pass + # Gather interested services + services = self.get_services_for_event(event) + if len(services) == 0: + return # no services need notifying + + # Do we know this user exists? If not, poke the user query API for + # all services which match that user regex. + + # Do we know this room alias exists? If not, poke the room alias query + # API for all services which match that room alias regex. + + # Fork off pushes to these services - XXX First cut, best effort diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 07ed0adcf8..277741fced 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -23,12 +23,6 @@ from ._base import SQLBaseStore logger = logging.getLogger(__name__) -namespace_enum = [ - "users", # 0 - "aliases", # 1 - "rooms" # 2 -] - # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): @@ -36,6 +30,10 @@ class ApplicationService(object): Provides methods to check if this service is "interested" in events. """ + NS_USERS = "users" + NS_ALIASES = "aliases" + NS_ROOMS = "rooms" + NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None): self.token = token @@ -52,7 +50,7 @@ class ApplicationService(object): if not namespaces: return None - for ns in ["users", "rooms", "aliases"]: + for ns in ApplicationService.NS_LIST: if type(namespaces[ns]) != list: raise ValueError("Bad namespace value for '%s'", ns) for regex in namespaces[ns]: @@ -68,31 +66,36 @@ class ApplicationService(object): def _matches_user(self, event): if (hasattr(event, "user_id") and - self._matches_regex(event.user_id, "users")): + self._matches_regex( + event.user_id, ApplicationService.NS_USERS)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member and hasattr(event, "state_key") - and self._matches_regex(event.state_key, "users")): + and self._matches_regex( + event.state_key, ApplicationService.NS_USERS)): return True return False def _matches_room_id(self, event): if hasattr(event, "room_id"): - return self._matches_regex(event.room_id, "rooms") + return self._matches_regex( + event.room_id, ApplicationService.NS_ROOMS + ) return False def _matches_aliases(self, event, alias_list): for alias in alias_list: - if self._matches_regex(alias, "aliases"): + if self._matches_regex(alias, ApplicationService.NS_ALIASES): return True return False - def is_interested(self, event, aliases_for_event=None): + def is_interested(self, event, restrict_to=None, aliases_for_event=None): """Check if this service is interested in this event. Args: event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. aliases_for_event(list): A list of all the known room aliases for this event. Returns: @@ -100,6 +103,9 @@ class ApplicationService(object): """ if aliases_for_event is None: aliases_for_event = [] + if restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so raise a general exception + raise Exception("Unexpected restrict_to value: %s". restrict_to) return (self._matches_user(event) or self._matches_aliases(event, aliases_for_event) @@ -215,7 +221,7 @@ class ApplicationServiceStore(SQLBaseStore): "DELETE FROM application_services_regex WHERE as_id=?", (as_id,) ) - for (ns_int, ns_str) in enumerate(namespace_enum): + for (ns_int, ns_str) in enumerate(ApplicationService.NS_LIST): if ns_str in service.namespaces: for regex in service.namespaces[ns_str]: txn.execute( @@ -285,9 +291,9 @@ class ApplicationServiceStore(SQLBaseStore): "url": res["url"], "token": as_token, "namespaces": { - "users": [], - "aliases": [], - "rooms": [] + ApplicationService.NS_USERS: [], + ApplicationService.NS_ALIASES: [], + ApplicationService.NS_ROOMS: [] } } # add the namespace regex if one exists @@ -295,7 +301,8 @@ class ApplicationServiceStore(SQLBaseStore): if ns_int is None: continue try: - services[as_token]["namespaces"][namespace_enum[ns_int]].append( + services[as_token]["namespaces"][ + ApplicationService.NS_LIST[ns_int]].append( res["regex"] ) except IndexError: -- cgit 1.4.1 From fed29251d70b050ea5799708b6b13be9617d1f6d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 13:23:58 +0000 Subject: Spelling --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9d7557f578..b9b2e25d1f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -767,7 +767,7 @@ class FederationHandler(BaseHandler): ) ) - logger.debug("on_query_auth reutrning: %s", ret) + logger.debug("on_query_auth returning: %s", ret) defer.returnValue(ret) -- cgit 1.4.1 From f2c039bfb958ed349bce42098e296995786374cc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 13:29:27 +0000 Subject: Implement restricted namespace checks. Begin fleshing out the main hook for notifying application services. --- synapse/handlers/appservice.py | 19 +++++++++++++++++++ synapse/storage/appservice.py | 21 ++++++++++++++++----- 2 files changed, 35 insertions(+), 5 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index bf68b33398..dac63e2245 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -17,6 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError +from synapse.storage.appservice import ApplicationService import logging @@ -88,8 +89,26 @@ class ApplicationServicesHandler(BaseHandler): # Do we know this user exists? If not, poke the user query API for # all services which match that user regex. + unknown_user = False # TODO check + if unknown_user: + user_query_services = self.get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_USERS + ) + for user_service in user_query_services: + pass # TODO poke User Query API # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. + unknown_room_alias = False # TODO check + if unknown_room_alias: + alias_query_services = self.get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_ALIASES + ) + for alias_service in alias_query_services: + pass # TODO poke Room Alias Query API # Fork off pushes to these services - XXX First cut, best effort + for service in services: + pass # TODO push event to service diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 277741fced..cdf26ee434 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -33,6 +33,9 @@ class ApplicationService(object): NS_USERS = "users" NS_ALIASES = "aliases" NS_ROOMS = "rooms" + # The ordering here is important as it is used to map database values (which + # are stored as ints representing the position in this list) to namespace + # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None): @@ -103,13 +106,21 @@ class ApplicationService(object): """ if aliases_for_event is None: aliases_for_event = [] - if restrict_to not in ApplicationService.NS_LIST: - # this is a programming error, so raise a general exception + if restrict_to and restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so fail early and raise a general + # exception raise Exception("Unexpected restrict_to value: %s". restrict_to) - return (self._matches_user(event) - or self._matches_aliases(event, aliases_for_event) - or self._matches_room_id(event)) + if not restrict_to: + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) + elif restrict_to == ApplicationService.NS_ALIASES: + return self._matches_aliases(event, aliases_for_event) + elif restrict_to == ApplicationService.NS_ROOMS: + return self._matches_room_id(event) + elif restrict_to == ApplicationService.NS_USERS: + return self._matches_user(event) def __str__(self): return "ApplicationService: %s" % (self.__dict__,) -- cgit 1.4.1 From 77a076bd25fc355c49251bcf489c0511c0f0f0af Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 13:35:17 +0000 Subject: Set combinations is | and not + --- synapse/handlers/federation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b9b2e25d1f..653ab0dbfa 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -114,9 +114,9 @@ class FederationHandler(BaseHandler): event_ids = set() if state: - event_ids += {e.event_id for e in state} + event_ids |= {e.event_id for e in state} if auth_chain: - event_ids += {e.event_id for e in auth_chain} + event_ids |= {e.event_id for e in auth_chain} seen_ids = (yield self.store.have_events(event_ids)).keys() -- cgit 1.4.1 From 6efd4d1649a539ba4bf1c884ebb90a48c2d1c8df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 13:57:54 +0000 Subject: Don't completely die if get auth_chain or querying auth_chain requests fail --- synapse/handlers/federation.py | 135 ++++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 63 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 653ab0dbfa..6727155c38 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -787,41 +787,45 @@ class FederationHandler(BaseHandler): if missing_auth: logger.debug("Missing auth: %s", missing_auth) # If we don't have all the auth events, we need to get them. - remote_auth_chain = yield self.replication_layer.get_event_auth( - origin, event.room_id, event.event_id - ) + try: + remote_auth_chain = yield self.replication_layer.get_event_auth( + origin, event.room_id, event.event_id + ) - seen_remotes = yield self.store.have_events( - [e.event_id for e in remote_auth_chain] - ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in remote_auth_chain] + ) - for e in remote_auth_chain: - if e.event_id in seen_remotes.keys(): - continue + for e in remote_auth_chain: + if e.event_id in seen_remotes.keys(): + continue - if e.event_id == event.event_id: - continue + if e.event_id == event.event_id: + continue - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in remote_auth_chain - if e.event_id in auth_ids - } - e.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in remote_auth_chain + if e.event_id in auth_ids + } + e.internal_metadata.outlier = True - logger.debug( - "do_auth %s missing_auth: %s", - event.event_id, e.event_id - ) - yield self._handle_new_event( - origin, e, auth_events=auth - ) + logger.debug( + "do_auth %s missing_auth: %s", + event.event_id, e.event_id + ) + yield self._handle_new_event( + origin, e, auth_events=auth + ) - if e.event_id in event_auth_events: - auth_events[(e.type, e.state_key)] = e - except AuthError: - pass + if e.event_id in event_auth_events: + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass + except: + # FIXME: + logger.exception("Failed to get auth chain") # FIXME: Assumes we have and stored all the state for all the # prev_events @@ -836,47 +840,52 @@ class FederationHandler(BaseHandler): auth_ids = self.auth.compute_auth_events(event, context) local_auth_chain = yield self.store.get_auth_chain(auth_ids) - # 2. Get remote difference. - result = yield self.replication_layer.query_auth( - origin, - event.room_id, - event.event_id, - local_auth_chain, - ) + try: + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) - seen_remotes = yield self.store.have_events( - [e.event_id for e in result["auth_chain"]] - ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) - # 3. Process any remote auth chain events we haven't seen. - for ev in result["auth_chain"]: - if ev.event_id in seen_remotes.keys(): - continue + # 3. Process any remote auth chain events we haven't seen. + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue - if ev.event_id == event.event_id: - continue + if ev.event_id == event.event_id: + continue - try: - auth_ids = [e_id for e_id, _ in ev.auth_events] - auth = { - (e.type, e.state_key): e for e in result["auth_chain"] - if e.event_id in auth_ids - } - ev.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in ev.auth_events] + auth = { + (e.type, e.state_key): e for e in result["auth_chain"] + if e.event_id in auth_ids + } + ev.internal_metadata.outlier = True - logger.debug( - "do_auth %s different_auth: %s", - event.event_id, e.event_id - ) + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) - yield self._handle_new_event( - origin, ev, auth_events=auth - ) + yield self._handle_new_event( + origin, ev, auth_events=auth + ) + + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev + except AuthError: + pass - if ev.event_id in event_auth_events: - auth_events[(ev.type, ev.state_key)] = ev - except AuthError: - pass + except: + # FIXME: + logger.exception("Failed to query auth chain") # 4. Look at rejects and their proofs. # TODO. -- cgit 1.4.1 From 94a5db9f4d400a345c5d8b9f7bacb0c9ccf99959 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 14:44:16 +0000 Subject: Add appservice package and move ApplicationService into it. --- synapse/appservice/__init__.py | 119 +++++++++++++++++++++++++++++++++++++++++ synapse/appservice/api.py | 15 ++++++ synapse/handlers/appservice.py | 4 +- synapse/storage/appservice.py | 105 +----------------------------------- 4 files changed, 138 insertions(+), 105 deletions(-) create mode 100644 synapse/appservice/__init__.py create mode 100644 synapse/appservice/api.py (limited to 'synapse/handlers') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py new file mode 100644 index 0000000000..f801fb5324 --- /dev/null +++ b/synapse/appservice/__init__.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.api.constants import EventTypes + +import re + + +class ApplicationService(object): + """Defines an application service. This definition is mostly what is + provided to the /register AS API. + + Provides methods to check if this service is "interested" in events. + """ + NS_USERS = "users" + NS_ALIASES = "aliases" + NS_ROOMS = "rooms" + # The ordering here is important as it is used to map database values (which + # are stored as ints representing the position in this list) to namespace + # values. + NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] + + def __init__(self, token, url=None, namespaces=None): + self.token = token + self.url = url + self.namespaces = self._check_namespaces(namespaces) + + def _check_namespaces(self, namespaces): + # Sanity check that it is of the form: + # { + # users: ["regex",...], + # aliases: ["regex",...], + # rooms: ["regex",...], + # } + if not namespaces: + return None + + for ns in ApplicationService.NS_LIST: + if type(namespaces[ns]) != list: + raise ValueError("Bad namespace value for '%s'", ns) + for regex in namespaces[ns]: + if not isinstance(regex, basestring): + raise ValueError("Expected string regex for ns '%s'", ns) + return namespaces + + def _matches_regex(self, test_string, namespace_key): + for regex in self.namespaces[namespace_key]: + if re.match(regex, test_string): + return True + return False + + def _matches_user(self, event): + if (hasattr(event, "user_id") and + self._matches_regex( + event.user_id, ApplicationService.NS_USERS)): + return True + # also check m.room.member state key + if (hasattr(event, "type") and event.type == EventTypes.Member + and hasattr(event, "state_key") + and self._matches_regex( + event.state_key, ApplicationService.NS_USERS)): + return True + return False + + def _matches_room_id(self, event): + if hasattr(event, "room_id"): + return self._matches_regex( + event.room_id, ApplicationService.NS_ROOMS + ) + return False + + def _matches_aliases(self, event, alias_list): + for alias in alias_list: + if self._matches_regex(alias, ApplicationService.NS_ALIASES): + return True + return False + + def is_interested(self, event, restrict_to=None, aliases_for_event=None): + """Check if this service is interested in this event. + + Args: + event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. + aliases_for_event(list): A list of all the known room aliases for + this event. + Returns: + bool: True if this service would like to know about this event. + """ + if aliases_for_event is None: + aliases_for_event = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so fail early and raise a general + # exception + raise Exception("Unexpected restrict_to value: %s". restrict_to) + + if not restrict_to: + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) + elif restrict_to == ApplicationService.NS_ALIASES: + return self._matches_aliases(event, aliases_for_event) + elif restrict_to == ApplicationService.NS_ROOMS: + return self._matches_room_id(event) + elif restrict_to == ApplicationService.NS_USERS: + return self._matches_user(event) + + def __str__(self): + return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py new file mode 100644 index 0000000000..803f97ea4f --- /dev/null +++ b/synapse/appservice/api.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index dac63e2245..f05b57bcb9 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError -from synapse.storage.appservice import ApplicationService +from synapse.appservice import ApplicationService import logging @@ -96,6 +96,7 @@ class ApplicationServicesHandler(BaseHandler): restrict_to=ApplicationService.NS_USERS ) for user_service in user_query_services: + # this needs to block XXX: Need to feed response back to caller pass # TODO poke User Query API # Do we know this room alias exists? If not, poke the room alias query @@ -107,6 +108,7 @@ class ApplicationServicesHandler(BaseHandler): restrict_to=ApplicationService.NS_ALIASES ) for alias_service in alias_query_services: + # this needs to block XXX: Need to feed response back to caller pass # TODO poke Room Alias Query API # Fork off pushes to these services - XXX First cut, best effort diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index cdf26ee434..48bc7e0fe6 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,119 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import re from twisted.internet import defer -from synapse.api.constants import EventTypes from synapse.api.errors import StoreError +from synapse.appservice import ApplicationService from ._base import SQLBaseStore logger = logging.getLogger(__name__) -# XXX: This feels like it should belong in a "models" module, not storage. -class ApplicationService(object): - """Defines an application service. - - Provides methods to check if this service is "interested" in events. - """ - NS_USERS = "users" - NS_ALIASES = "aliases" - NS_ROOMS = "rooms" - # The ordering here is important as it is used to map database values (which - # are stored as ints representing the position in this list) to namespace - # values. - NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - - def __init__(self, token, url=None, namespaces=None): - self.token = token - self.url = url - self.namespaces = self._check_namespaces(namespaces) - - def _check_namespaces(self, namespaces): - # Sanity check that it is of the form: - # { - # users: ["regex",...], - # aliases: ["regex",...], - # rooms: ["regex",...], - # } - if not namespaces: - return None - - for ns in ApplicationService.NS_LIST: - if type(namespaces[ns]) != list: - raise ValueError("Bad namespace value for '%s'", ns) - for regex in namespaces[ns]: - if not isinstance(regex, basestring): - raise ValueError("Expected string regex for ns '%s'", ns) - return namespaces - - def _matches_regex(self, test_string, namespace_key): - for regex in self.namespaces[namespace_key]: - if re.match(regex, test_string): - return True - return False - - def _matches_user(self, event): - if (hasattr(event, "user_id") and - self._matches_regex( - event.user_id, ApplicationService.NS_USERS)): - return True - # also check m.room.member state key - if (hasattr(event, "type") and event.type == EventTypes.Member - and hasattr(event, "state_key") - and self._matches_regex( - event.state_key, ApplicationService.NS_USERS)): - return True - return False - - def _matches_room_id(self, event): - if hasattr(event, "room_id"): - return self._matches_regex( - event.room_id, ApplicationService.NS_ROOMS - ) - return False - - def _matches_aliases(self, event, alias_list): - for alias in alias_list: - if self._matches_regex(alias, ApplicationService.NS_ALIASES): - return True - return False - - def is_interested(self, event, restrict_to=None, aliases_for_event=None): - """Check if this service is interested in this event. - - Args: - event(Event): The event to check. - restrict_to(str): The namespace to restrict regex tests to. - aliases_for_event(list): A list of all the known room aliases for - this event. - Returns: - bool: True if this service would like to know about this event. - """ - if aliases_for_event is None: - aliases_for_event = [] - if restrict_to and restrict_to not in ApplicationService.NS_LIST: - # this is a programming error, so fail early and raise a general - # exception - raise Exception("Unexpected restrict_to value: %s". restrict_to) - - if not restrict_to: - return (self._matches_user(event) - or self._matches_aliases(event, aliases_for_event) - or self._matches_room_id(event)) - elif restrict_to == ApplicationService.NS_ALIASES: - return self._matches_aliases(event, aliases_for_event) - elif restrict_to == ApplicationService.NS_ROOMS: - return self._matches_room_id(event) - elif restrict_to == ApplicationService.NS_USERS: - return self._matches_user(event) - - def __str__(self): - return "ApplicationService: %s" % (self.__dict__,) - - class ApplicationServiceCache(object): """Caches ApplicationServices and provides utility functions on top. -- cgit 1.4.1 From 7dd1c5c542d58464085b11ababe746c6a60515ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 16:12:04 +0000 Subject: Neaten the handling of state and auth_chain up a bit --- synapse/handlers/federation.py | 57 ++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 27 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6727155c38..86953bf8c8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -30,6 +30,7 @@ from synapse.types import UserID from twisted.internet import defer +import itertools import logging @@ -112,14 +113,6 @@ class FederationHandler(BaseHandler): logger.debug("Event: %s", event) - event_ids = set() - if state: - event_ids |= {e.event_id for e in state} - if auth_chain: - event_ids |= {e.event_id for e in auth_chain} - - seen_ids = (yield self.store.have_events(event_ids)).keys() - # FIXME (erikj): Awful hack to make the case where we are not currently # in the room work current_state = None @@ -131,27 +124,37 @@ class FederationHandler(BaseHandler): logger.debug("Got event for room we're not in.") current_state = state + event_ids = set() + if state: + event_ids |= {e.event_id for e in state} + if auth_chain: + event_ids |= {e.event_id for e in auth_chain} + + seen_ids = (yield self.store.have_events(event_ids)).keys() + if state and auth_chain is not None: - for list_of_pdus in [auth_chain, state]: - for e in list_of_pdus: - if e.event_id in seen_ids: - continue + # If we have any state or auth_chain given to us by the replication + # layer, then we should handle them (if we haven't before.) + for e in itertools.chain(auth_chain, state): + if e.event_id in seen_ids: + continue - e.internal_metadata.outlier = True - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - yield self._handle_new_event( - origin, e, auth_events=auth - ) - except: - logger.exception( - "Failed to handle state event %s", - e.event_id, - ) + e.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + seen_ids.add(e.event_id) + except: + logger.exception( + "Failed to handle state event %s", + e.event_id, + ) try: yield self._handle_new_event( -- cgit 1.4.1 From c0462dbf1533f285f632dcb0a74c0ef0c3e2475b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 10:16:51 +0000 Subject: Rearrange persist_event so that do all the queries that need to be done before returning early if we have already persisted that event. --- synapse/events/__init__.py | 2 +- synapse/handlers/federation.py | 2 + synapse/storage/__init__.py | 145 +++++++++++++++++++++-------------------- 3 files changed, 77 insertions(+), 72 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index bf07951027..8f0c6e959f 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -77,7 +77,7 @@ class EventBase(object): return self.content["membership"] def is_state(self): - return hasattr(self, "state_key") + return hasattr(self, "state_key") and self.state_key is not None def get_dict(self): d = dict(self._event_dict) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 86953bf8c8..0876589e31 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -515,6 +515,8 @@ class FederationHandler(BaseHandler): "Failed to get destination from event %s", s.event_id ) + destinations.remove(origin) + logger.debug( "on_send_join_request: Sending event: %s, signatures: %s", event.event_id, diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 93ab26fcd1..30ce378900 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -163,19 +163,70 @@ class DataStore(RoomMemberStore, RoomStore, def _persist_event_txn(self, txn, event, context, backfilled, stream_ordering=None, is_new_state=True, current_state=None): - if event.type == EventTypes.Member: - self._store_room_member_txn(txn, event) - elif event.type == EventTypes.Feedback: - self._store_feedback_txn(txn, event) - elif event.type == EventTypes.Name: - self._store_room_name_txn(txn, event) - elif event.type == EventTypes.Topic: - self._store_room_topic_txn(txn, event) - elif event.type == EventTypes.Redaction: - self._store_redaction(txn, event) + + # We purposefully do this first since if we include a `current_state` + # key, we *want* to update the `current_state_events` table + if current_state: + txn.execute( + "DELETE FROM current_state_events WHERE room_id = ?", + (event.room_id,) + ) + + for s in current_state: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": s.event_id, + "room_id": s.room_id, + "type": s.type, + "state_key": s.state_key, + }, + or_replace=True, + ) + + if event.is_state() and is_new_state: + if not backfilled and not context.rejected: + self._simple_insert_txn( + txn, + table="state_forward_extremities", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + or_replace=True, + ) + + for prev_state_id, _ in event.prev_state: + self._simple_delete_txn( + txn, + table="state_forward_extremities", + keyvalues={ + "event_id": prev_state_id, + } + ) outlier = event.internal_metadata.is_outlier() + if not outlier: + self._store_state_groups_txn(txn, event, context) + + self._update_min_depth_for_room_txn( + txn, + event.room_id, + event.depth + ) + + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + have_persisted = self._simple_select_one_onecol_txn( txn, table="event_json", @@ -209,6 +260,17 @@ class DataStore(RoomMemberStore, RoomStore, ) return + if event.type == EventTypes.Member: + self._store_room_member_txn(txn, event) + elif event.type == EventTypes.Feedback: + self._store_feedback_txn(txn, event) + elif event.type == EventTypes.Name: + self._store_room_name_txn(txn, event) + elif event.type == EventTypes.Topic: + self._store_room_topic_txn(txn, event) + elif event.type == EventTypes.Redaction: + self._store_redaction(txn, event) + event_dict = { k: v for k, v in event.get_dict().items() @@ -273,41 +335,10 @@ class DataStore(RoomMemberStore, RoomStore, ) raise _RollbackButIsFineException("_persist_event") - self._handle_prev_events( - txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, - ) - - if not outlier: - self._store_state_groups_txn(txn, event, context) - if context.rejected: self._store_rejections_txn(txn, event.event_id, context.rejected) - if current_state: - txn.execute( - "DELETE FROM current_state_events WHERE room_id = ?", - (event.room_id,) - ) - - for s in current_state: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": s.event_id, - "room_id": s.room_id, - "type": s.type, - "state_key": s.state_key, - }, - or_replace=True, - ) - - is_state = hasattr(event, "state_key") and event.state_key is not None - if is_state: + if event.is_state(): vals = { "event_id": event.event_id, "room_id": event.room_id, @@ -315,6 +346,7 @@ class DataStore(RoomMemberStore, RoomStore, "state_key": event.state_key, } + # TODO: How does this work with backfilling? if hasattr(event, "replaces_state"): vals["prev_state"] = event.replaces_state @@ -351,28 +383,6 @@ class DataStore(RoomMemberStore, RoomStore, or_ignore=True, ) - if not backfilled and not context.rejected: - self._simple_insert_txn( - txn, - table="state_forward_extremities", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - or_replace=True, - ) - - for prev_state_id, _ in event.prev_state: - self._simple_delete_txn( - txn, - table="state_forward_extremities", - keyvalues={ - "event_id": prev_state_id, - } - ) - for hash_alg, hash_base64 in event.hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_event_content_hash_txn( @@ -403,13 +413,6 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, ref_alg, ref_hash_bytes ) - if not outlier: - self._update_min_depth_for_room_txn( - txn, - event.room_id, - event.depth - ) - def _store_redaction(self, txn, event): txn.execute( "INSERT OR IGNORE INTO redactions " -- cgit 1.4.1 From 17753f0c20d0d8190095c5a3183630b78bf9650c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 11:19:18 +0000 Subject: Add stub ApplicationServiceApi and glue it with the handler. --- synapse/appservice/__init__.py | 3 ++- synapse/appservice/api.py | 21 +++++++++++++++++++++ synapse/handlers/appservice.py | 18 +++++++++++++++--- synapse/storage/appservice.py | 1 + 4 files changed, 39 insertions(+), 4 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f801fb5324..92f64619c9 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -31,10 +31,11 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - def __init__(self, token, url=None, namespaces=None): + def __init__(self, token, url=None, namespaces=None, txn_id=None): self.token = token self.url = url self.namespaces = self._check_namespaces(namespaces) + self.txn_id = None def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 803f97ea4f..158aded66e 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,3 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. + +class ApplicationServiceApi(object): + """This class manages HS -> AS communications, including querying and + pushing. + """ + + def __init__(self, hs): + self.hs_token = "_hs_token_" # TODO extract hs token + + def query_user(self, service, user_id): + pass + + def query_alias(self, service, alias): + pass + + def push_bulk(self, service, events): + pass + + def push(self, service, event): + pass + diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index f05b57bcb9..9cdeaa2d94 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService +from synapse.appservice.api import ApplicationServiceApi import logging @@ -29,6 +30,7 @@ class ApplicationServicesHandler(BaseHandler): def __init__(self, hs): super(ApplicationServicesHandler, self).__init__(hs) + self.appservice_api = ApplicationServiceApi(hs) @defer.inlineCallbacks def register(self, app_service): @@ -97,7 +99,12 @@ class ApplicationServicesHandler(BaseHandler): ) for user_service in user_query_services: # this needs to block XXX: Need to feed response back to caller - pass # TODO poke User Query API + is_known_user = self.appservice_api.query_user( + user_service, event + ) + if is_known_user: + # the user exists now,so don't query more ASes. + break # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. @@ -109,8 +116,13 @@ class ApplicationServicesHandler(BaseHandler): ) for alias_service in alias_query_services: # this needs to block XXX: Need to feed response back to caller - pass # TODO poke Room Alias Query API + is_known_alias = self.appservice_api.query_alias( + alias_service, event + ) + if is_known_alias: + # the alias exists now so don't query more ASes. + break # Fork off pushes to these services - XXX First cut, best effort for service in services: - pass # TODO push event to service + self.appservice_api.push(service, event) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 48bc7e0fe6..abb617f049 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -216,6 +216,7 @@ class ApplicationServiceStore(SQLBaseStore): except IndexError: logger.error("Bad namespace enum '%s'. %s", ns_int, res) + # TODO get last successful txn id f.e. service for service in services.values(): logger.info("Found application service: %s", service) self.cache.services.append(ApplicationService( -- cgit 1.4.1 From 525a218b2b072b24721c9c9efae42aae21388fc8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 12:24:20 +0000 Subject: Begin to add unit tests for appservice glue and regex testing. --- synapse/appservice/__init__.py | 14 ++++++-- synapse/handlers/appservice.py | 8 +++-- tests/appservice/__init__.py | 14 ++++++++ tests/appservice/test_appservice.py | 58 +++++++++++++++++++++++++++++++ tests/handlers/test_appservice.py | 68 +++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 tests/appservice/__init__.py create mode 100644 tests/appservice/test_appservice.py create mode 100644 tests/handlers/test_appservice.py (limited to 'synapse/handlers') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 92f64619c9..0c7f58574e 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -14,8 +14,11 @@ # limitations under the License. from synapse.api.constants import EventTypes +import logging import re +logger = logging.getLogger(__name__) + class ApplicationService(object): """Defines an application service. This definition is mostly what is @@ -56,15 +59,22 @@ class ApplicationService(object): return namespaces def _matches_regex(self, test_string, namespace_key): + if not isinstance(test_string, basestring): + logger.warning( + "Expected a string to test regex against, but got %s", + test_string + ) + return False + for regex in self.namespaces[namespace_key]: if re.match(regex, test_string): return True return False def _matches_user(self, event): - if (hasattr(event, "user_id") and + if (hasattr(event, "sender") and self._matches_regex( - event.user_id, ApplicationService.NS_USERS)): + event.sender, ApplicationService.NS_USERS)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 9cdeaa2d94..3188c60f3d 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -26,10 +26,14 @@ import logging logger = logging.getLogger(__name__) -class ApplicationServicesHandler(BaseHandler): +# NB: Purposefully not inheriting BaseHandler since that contains way too much +# setup code which this handler does not need or use. This makes testing a lot +# easier. +class ApplicationServicesHandler(object): def __init__(self, hs): - super(ApplicationServicesHandler, self).__init__(hs) + self.store = hs.get_datastore() + self.hs = hs self.appservice_api = ApplicationServiceApi(hs) @defer.inlineCallbacks diff --git a/tests/appservice/__init__.py b/tests/appservice/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/tests/appservice/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py new file mode 100644 index 0000000000..5cfd26daa6 --- /dev/null +++ b/tests/appservice/test_appservice.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.appservice import ApplicationService + +from mock import Mock, PropertyMock +from tests import unittest + + +class ApplicationServiceTestCase(unittest.TestCase): + + def setUp(self): + self.service = ApplicationService( + url="some_url", + token="some_token", + namespaces={ + ApplicationService.NS_USERS: [], + ApplicationService.NS_ROOMS: [], + ApplicationService.NS_ALIASES: [] + } + ) + self.event = Mock( + type="m.something", room_id="!foo:bar", sender="@someone:somewhere" + ) + + def test_regex_user_id_prefix_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_user_id_prefix_no_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.assertFalse(self.service.is_interested(self.event)) + + def test_regex_room_member_is_checked(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.event.type = "m.room.member" + self.event.state_key = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py new file mode 100644 index 0000000000..9c464e7fbc --- /dev/null +++ b/tests/handlers/test_appservice.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer +from .. import unittest + +from synapse.handlers.appservice import ApplicationServicesHandler + +from collections import namedtuple +from mock import Mock + +# TODO: Should this be a more general thing? tests/api/test_filtering.py uses it +MockEvent = namedtuple("MockEvent", "sender type room_id") + + +class AppServiceHandlerTestCase(unittest.TestCase): + """ Tests the ApplicationServicesHandler. """ + + def setUp(self): + self.mock_store = Mock() + self.mock_as_api = Mock() + hs = Mock() + hs.get_datastore = Mock(return_value=self.mock_store) + self.handler = ApplicationServicesHandler(hs) # thing being tested + + # FIXME Would be nice to DI this rather than monkey patch:( + if not hasattr(self.handler, "appservice_api"): + # someone probably updated the handler but not the tests. Fail fast. + raise Exception("Test expected handler.appservice_api to exist.") + self.handler.appservice_api = self.mock_as_api + + def test_notify_interested_services(self): + interested_service = self._mkservice(is_interested=True) + services = [ + self._mkservice(is_interested=False), + interested_service, + self._mkservice(is_interested=False) + ] + + self.mock_store.get_app_services = Mock(return_value=services) + + event = MockEvent( + sender="@someone:anywhere", + type="m.room.message", + room_id="!foo:bar" + ) + self.mock_as_api.push = Mock() + self.handler.notify_interested_services(event) + self.mock_as_api.push.assert_called_once_with(interested_service, event) + + def _mkservice(self, is_interested): + service = Mock() + service.is_interested = Mock(return_value=is_interested) + service.token = "mock_service_token" + service.url = "mock_service_url" + return service -- cgit 1.4.1 From 650e32d45580ddd13826364291bda6760c014df9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 14:06:42 +0000 Subject: Change context.auth_events to what the auth_events would be bases on context.current_state, rather than based on the auth_events from the event. --- synapse/api/auth.py | 12 ++++++------ synapse/handlers/federation.py | 4 +++- synapse/state.py | 8 ++++++-- 3 files changed, 15 insertions(+), 9 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3471afd7e7..7105ee21dc 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -358,7 +358,7 @@ class Auth(object): def add_auth_events(self, builder, context): yield run_on_reactor() - auth_ids = self.compute_auth_events(builder, context) + auth_ids = self.compute_auth_events(builder, context.current_state) auth_events_entries = yield self.store.add_event_hashes( auth_ids @@ -372,26 +372,26 @@ class Auth(object): if v.event_id in auth_ids } - def compute_auth_events(self, event, context): + def compute_auth_events(self, event, current_state): if event.type == EventTypes.Create: return [] auth_ids = [] key = (EventTypes.PowerLevels, "", ) - power_level_event = context.current_state.get(key) + power_level_event = current_state.get(key) if power_level_event: auth_ids.append(power_level_event.event_id) key = (EventTypes.JoinRules, "", ) - join_rule_event = context.current_state.get(key) + join_rule_event = current_state.get(key) key = (EventTypes.Member, event.user_id, ) - member_event = context.current_state.get(key) + member_event = current_state.get(key) key = (EventTypes.Create, "", ) - create_event = context.current_state.get(key) + create_event = current_state.get(key) if create_event: auth_ids.append(create_event.event_id) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0876589e31..2e2c23ef63 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -842,7 +842,9 @@ class FederationHandler(BaseHandler): logger.debug("Different auth: %s", different_auth) # 1. Get what we think is the auth chain. - auth_ids = self.auth.compute_auth_events(event, context) + auth_ids = self.auth.compute_auth_events( + event, context.current_state + ) local_auth_chain = yield self.store.get_auth_chain(auth_ids) try: diff --git a/synapse/state.py b/synapse/state.py index 6a6fb8aea0..695a5e7ac4 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -103,7 +103,9 @@ class StateHandler(object): context.state_group = None if hasattr(event, "auth_events") and event.auth_events: - auth_ids = zip(*event.auth_events)[0] + auth_ids = self.hs.get_auth().compute_auth_events( + event, context.current_state + ) context.auth_events = { k: v for k, v in context.current_state.items() @@ -149,7 +151,9 @@ class StateHandler(object): event.unsigned["replaces_state"] = replaces.event_id if hasattr(event, "auth_events") and event.auth_events: - auth_ids = zip(*event.auth_events)[0] + auth_ids = self.hs.get_auth().compute_auth_events( + event, context.current_state + ) context.auth_events = { k: v for k, v in context.current_state.items() -- cgit 1.4.1 From 95e2d2d36d6dfb7205c40fa8e59ef350f8096395 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 15:02:23 +0000 Subject: When returning lists of servers from alias lookups, put the current server first in the list --- synapse/handlers/directory.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 58e9a91562..7b60921040 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -113,7 +113,16 @@ class DirectoryHandler(BaseHandler): ) extra_servers = yield self.store.get_joined_hosts_for_room(room_id) - servers = list(set(extra_servers) | set(servers)) + servers = set(extra_servers) | set(servers) + + # If this server is in the list of servers, return it first. + if self.server_name in servers: + servers = ( + [self.server_name] + + [s for s in servers if s != self.server_name] + ) + else: + servers = list(servers) defer.returnValue({ "room_id": room_id, -- cgit 1.4.1 From ae46f10fc5dba0f81518e3144ab8d9ed7a7d03bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 16:28:12 +0000 Subject: Apply sanity to the transport client interface. Convert 'make_join' and 'send_join' to accept iterables of destinations --- synapse/api/errors.py | 8 +++- synapse/federation/federation_client.py | 82 ++++++++++++++++++++------------- synapse/federation/transaction_queue.py | 23 +++++++-- synapse/federation/transport/client.py | 42 +++++++---------- synapse/handlers/federation.py | 4 +- synapse/http/matrixfederationclient.py | 42 ++++++++++++++--- 6 files changed, 130 insertions(+), 71 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index ad478aa6b7..5041828f18 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -39,7 +39,7 @@ class Codes(object): TOO_LARGE = "M_TOO_LARGE" -class CodeMessageException(Exception): +class CodeMessageException(RuntimeError): """An exception with integer code and message string attributes.""" def __init__(self, code, msg): @@ -227,3 +227,9 @@ class FederationError(RuntimeError): "affected": self.affected, "source": self.source if self.source else self.affected, } + + +class HttpResponseException(CodeMessageException): + def __init__(self, code, msg, response): + self.response = response + super(HttpResponseException, self).__init__(code, msg) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index d6b8c43916..eb36ec040b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -19,6 +19,7 @@ from twisted.internet import defer from .federation_base import FederationBase from .units import Edu +from synapse.api.errors import CodeMessageException from synapse.util.logutils import log_function from synapse.events import FrozenEvent @@ -180,7 +181,8 @@ class FederationClient(FederationBase): pdu = yield self._check_sigs_and_hash(pdu) break - + except CodeMessageException: + raise except Exception as e: logger.info( "Failed to get PDU %s from %s because %s", @@ -264,45 +266,63 @@ class FederationClient(FederationBase): defer.returnValue(self.event_from_pdu_json(pdu_dict)) break - except Exception as e: - logger.warn("Failed to make_join via %s", destination) + except CodeMessageException: + raise + except RuntimeError as e: + logger.warn( + "Failed to make_join via %s: %s", + destination, e.message + ) + + raise RuntimeError("Failed to send to any server.") @defer.inlineCallbacks - def send_join(self, destination, pdu): - time_now = self._clock.time_msec() - _, content = yield self.transport_layer.send_join( - destination=destination, - room_id=pdu.room_id, - event_id=pdu.event_id, - content=pdu.get_pdu_json(time_now), - ) + def send_join(self, destinations, pdu): + for destination in destinations: + try: + time_now = self._clock.time_msec() + _, content = yield self.transport_layer.send_join( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=pdu.get_pdu_json(time_now), + ) - logger.debug("Got content: %s", content) + logger.debug("Got content: %s", content) - state = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("state", []) - ] + state = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("state", []) + ] - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("auth_chain", []) - ] + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("auth_chain", []) + ] - signed_state = yield self._check_sigs_and_hash_and_fetch( - destination, state, outlier=True - ) + signed_state = yield self._check_sigs_and_hash_and_fetch( + destination, state, outlier=True + ) - signed_auth = yield self._check_sigs_and_hash_and_fetch( - destination, auth_chain, outlier=True - ) + signed_auth = yield self._check_sigs_and_hash_and_fetch( + destination, auth_chain, outlier=True + ) - auth_chain.sort(key=lambda e: e.depth) + auth_chain.sort(key=lambda e: e.depth) + + defer.returnValue({ + "state": signed_state, + "auth_chain": signed_auth, + }) + except CodeMessageException: + raise + except RuntimeError as e: + logger.warn( + "Failed to send_join via %s: %s", + destination, e.message + ) - defer.returnValue({ - "state": signed_state, - "auth_chain": signed_auth, - }) + raise RuntimeError("Failed to send to any server.") @defer.inlineCallbacks def send_invite(self, destination, room_id, event_id, pdu): diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 9d4f2c09a2..f38aeba7cc 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -19,6 +19,7 @@ from twisted.internet import defer from .persistence import TransactionActions from .units import Transaction +from synapse.api.errors import HttpResponseException from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext @@ -238,9 +239,14 @@ class TransactionQueue(object): del p["age_ts"] return data - code, response = yield self.transport_layer.send_transaction( - transaction, json_data_cb - ) + try: + response = yield self.transport_layer.send_transaction( + transaction, json_data_cb + ) + code = 200 + except HttpResponseException as e: + code = e.code + response = e.response logger.info("TX [%s] got %d response", destination, code) @@ -274,8 +280,7 @@ class TransactionQueue(object): pass logger.debug("TX [%s] Yielded to callbacks", destination) - - except Exception as e: + except RuntimeError as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. logger.warn( @@ -283,6 +288,14 @@ class TransactionQueue(object): destination, e, ) + except Exception as e: + # We capture this here as there as nothing actually listens + # for this finishing functions deferred. + logger.exception( + "TX [%s] Problem in _attempt_transaction: %s", + destination, + e, + ) self.set_retrying(destination, retry_interval) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 4cb1dea2de..8b137e7128 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -19,7 +19,6 @@ from synapse.api.urls import FEDERATION_PREFIX as PREFIX from synapse.util.logutils import log_function import logging -import json logger = logging.getLogger(__name__) @@ -129,7 +128,7 @@ class TransportLayerClient(object): # generated by the json_data_callback. json_data = transaction.get_dict() - code, response = yield self.client.put_json( + response = yield self.client.put_json( transaction.destination, path=PREFIX + "/send/%s/" % transaction.transaction_id, data=json_data, @@ -137,95 +136,86 @@ class TransportLayerClient(object): ) logger.debug( - "send_data dest=%s, txid=%s, got response: %d", - transaction.destination, transaction.transaction_id, code + "send_data dest=%s, txid=%s, got response: 200", + transaction.destination, transaction.transaction_id, ) - defer.returnValue((code, response)) + defer.returnValue(response) @defer.inlineCallbacks @log_function def make_query(self, destination, query_type, args, retry_on_dns_fail): path = PREFIX + "/query/%s" % query_type - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, args=args, retry_on_dns_fail=retry_on_dns_fail, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def make_join(self, destination, room_id, user_id, retry_on_dns_fail=True): path = PREFIX + "/make_join/%s/%s" % (room_id, user_id) - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, retry_on_dns_fail=retry_on_dns_fail, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def send_join(self, destination, room_id, event_id, content): path = PREFIX + "/send_join/%s/%s" % (room_id, event_id) - code, content = yield self.client.put_json( + response = yield self.client.put_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_join", code) - - defer.returnValue(json.loads(content)) + defer.returnValue(response) @defer.inlineCallbacks @log_function def send_invite(self, destination, room_id, event_id, content): path = PREFIX + "/invite/%s/%s" % (room_id, event_id) - code, content = yield self.client.put_json( + response = yield self.client.put_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_invite", code) - - defer.returnValue(json.loads(content)) + defer.returnValue(response) @defer.inlineCallbacks @log_function def get_event_auth(self, destination, room_id, event_id): path = PREFIX + "/event_auth/%s/%s" % (room_id, event_id) - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def send_query_auth(self, destination, room_id, event_id, content): path = PREFIX + "/query_auth/%s/%s" % (room_id, event_id) - code, content = yield self.client.post_json( + content = yield self.client.post_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_invite", code) - - defer.returnValue(json.loads(content)) + defer.returnValue(content) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0876589e31..a968a87360 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -288,7 +288,7 @@ class FederationHandler(BaseHandler): logger.debug("Joining %s to %s", joinee, room_id) pdu = yield self.replication_layer.make_join( - target_host, + [target_host], room_id, joinee ) @@ -331,7 +331,7 @@ class FederationHandler(BaseHandler): new_event = builder.build() ret = yield self.replication_layer.send_join( - target_host, + [target_host], new_event ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index c7bf1b47b8..8559d06b7f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -27,7 +27,9 @@ from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json -from synapse.api.errors import CodeMessageException, SynapseError, Codes +from synapse.api.errors import ( + SynapseError, Codes, HttpResponseException, +) from syutil.crypto.jsonsign import sign_json @@ -163,13 +165,12 @@ class MatrixFederationHttpClient(object): ) if 200 <= response.code < 300: - # We need to update the transactions table to say it was sent? pass else: # :'( # Update transactions table? - raise CodeMessageException( - response.code, response.phrase + raise HttpResponseException( + response.code, response.phrase, response ) defer.returnValue(response) @@ -238,11 +239,20 @@ class MatrixFederationHttpClient(object): headers_dict={"Content-Type": ["application/json"]}, ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + logger.debug("Getting resp body") body = yield readBody(response) logger.debug("Got resp body") - defer.returnValue((response.code, body)) + defer.returnValue(json.loads(body)) @defer.inlineCallbacks def post_json(self, destination, path, data={}): @@ -275,11 +285,20 @@ class MatrixFederationHttpClient(object): headers_dict={"Content-Type": ["application/json"]}, ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + logger.debug("Getting resp body") body = yield readBody(response) logger.debug("Got resp body") - defer.returnValue((response.code, body)) + defer.returnValue(json.loads(body)) @defer.inlineCallbacks def get_json(self, destination, path, args={}, retry_on_dns_fail=True): @@ -321,7 +340,18 @@ class MatrixFederationHttpClient(object): retry_on_dns_fail=retry_on_dns_fail ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + + logger.debug("Getting resp body") body = yield readBody(response) + logger.debug("Got resp body") defer.returnValue(json.loads(body)) -- cgit 1.4.1 From aa8cce58bf3d3dbd1a0d512dbbd41b6545d2b1f9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 16:44:53 +0000 Subject: Add query_user/alias APIs. --- synapse/appservice/api.py | 47 ++++++++++++++++++++++++++++++++++++++---- synapse/handlers/appservice.py | 26 ++++++++++++++--------- 2 files changed, 59 insertions(+), 14 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 158aded66e..799ada96df 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -12,25 +12,64 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer +from twisted.web.client import PartialDownloadError +from synapse.http.client import SimpleHttpClient -class ApplicationServiceApi(object): +import logging +import urllib + +logger = logging.getLogger(__name__) + + +class ApplicationServiceApi(SimpleHttpClient): """This class manages HS -> AS communications, including querying and pushing. """ def __init__(self, hs): + super(ApplicationServiceApi, self).__init__(hs) self.hs_token = "_hs_token_" # TODO extract hs token + @defer.inlineCallbacks def query_user(self, service, user_id): - pass + uri = service.url + ("/users/%s" % urllib.quote(user_id)) + response = None + try: + response = yield self.get_json(uri, { + "access_token": self.hs_token + }) + if response: # just an empty json object + defer.returnValue(True) + except PartialDownloadError as e: + if e.status == 404: + defer.returnValue(False) + return + logger.warning("query_user to %s received %s", (uri, e.status)) + @defer.inlineCallbacks def query_alias(self, service, alias): - pass + uri = service.url + ("/rooms/%s" % urllib.quote(alias)) + response = None + try: + response = yield self.get_json(uri, { + "access_token": self.hs_token + }) + logger.info("%s", response[0]) + if response: # just an empty json object + defer.returnValue(True) + except PartialDownloadError as e: + if e.status == 404: + defer.returnValue(False) + return + logger.warning("query_alias to %s received %s", (uri, e.status)) def push_bulk(self, service, events): pass + @defer.inlineCallbacks def push(self, service, event): - pass + response = yield self.push_bulk(service, [event]) + defer.returnValue(response) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 3188c60f3d..dd860b2244 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -55,10 +55,14 @@ class ApplicationServicesHandler(object): logger.info("Updating application service info...") yield self.store.update_app_service(app_service) + logger.info("Sending ping to %s...", app_service.url) + yield self.appservice_api.query_alias(app_service, "ping") + def unregister(self, token): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) + @defer.inlineCallbacks def get_services_for_event(self, event, restrict_to=""): """Retrieve a list of application services interested in this event. @@ -71,14 +75,15 @@ class ApplicationServicesHandler(object): """ # We need to know the aliases associated with this event.room_id, if any alias_list = [] # TODO - + services = yield self.store.get_app_services() interested_list = [ - s for s in self.store.get_app_services() if ( + s for s in services if ( s.is_interested(event, restrict_to, alias_list) ) ] - return interested_list + defer.returnValue(interested_list) + @defer.inlineCallbacks def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. @@ -89,7 +94,7 @@ class ApplicationServicesHandler(object): event(Event): The event to push out to interested services. """ # Gather interested services - services = self.get_services_for_event(event) + services = yield self.get_services_for_event(event) if len(services) == 0: return # no services need notifying @@ -97,14 +102,14 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = False # TODO check if unknown_user: - user_query_services = self.get_services_for_event( + user_query_services = yield self.get_services_for_event( event=event, restrict_to=ApplicationService.NS_USERS ) for user_service in user_query_services: # this needs to block XXX: Need to feed response back to caller - is_known_user = self.appservice_api.query_user( - user_service, event + is_known_user = yield self.appservice_api.query_user( + user_service, event.sender ) if is_known_user: # the user exists now,so don't query more ASes. @@ -114,14 +119,15 @@ class ApplicationServicesHandler(object): # API for all services which match that room alias regex. unknown_room_alias = False # TODO check if unknown_room_alias: - alias_query_services = self.get_services_for_event( + alias = "something" # TODO + alias_query_services = yield self.get_services_for_event( event=event, restrict_to=ApplicationService.NS_ALIASES ) for alias_service in alias_query_services: # this needs to block XXX: Need to feed response back to caller - is_known_alias = self.appservice_api.query_alias( - alias_service, event + is_known_alias = yield self.appservice_api.query_alias( + alias_service, alias ) if is_known_alias: # the alias exists now so don't query more ASes. -- cgit 1.4.1 From 96d4bf90120a07faa5163c2e5af542358554dd61 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:07:31 +0000 Subject: Modify API for SimpleHttpClient.get_json and update usages. Previously, this would only return the HTTP body as JSON, and discard other response information (e.g. the HTTP response code). This has now been changed to throw a CodeMessageException on a non-2xx response, with the response code and body, which can then be parsed as JSON. Affected modules include: - Registration/Login (when using an email for IS auth) --- synapse/handlers/login.py | 33 +++++++++++++++----------- synapse/handlers/register.py | 56 ++++++++++++++++++++++++++------------------ synapse/http/client.py | 19 +++++++-------- 3 files changed, 61 insertions(+), 47 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index d297d71c03..7447800460 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -16,12 +16,13 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.errors import LoginError, Codes +from synapse.api.errors import LoginError, Codes, CodeMessageException from synapse.http.client import SimpleHttpClient from synapse.util.emailutils import EmailException import synapse.util.emailutils as emailutils import bcrypt +import json import logging logger = logging.getLogger(__name__) @@ -96,16 +97,20 @@ class LoginHandler(BaseHandler): @defer.inlineCallbacks def _query_email(self, email): - httpCli = SimpleHttpClient(self.hs) - data = yield httpCli.get_json( - # TODO FIXME This should be configurable. - # XXX: ID servers need to use HTTPS - "http://%s%s" % ( - "matrix.org:8090", "/_matrix/identity/api/v1/lookup" - ), - { - 'medium': 'email', - 'address': email - } - ) - defer.returnValue(data) + http_client = SimpleHttpClient(self.hs) + try: + data = yield http_client.get_json( + # TODO FIXME This should be configurable. + # XXX: ID servers need to use HTTPS + "http://%s%s" % ( + "matrix.org:8090", "/_matrix/identity/api/v1/lookup" + ), + { + 'medium': 'email', + 'address': email + } + ) + defer.returnValue(data) + except CodeMessageException as e: + data = json.loads(e.msg) + defer.returnValue(data) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 66a89c10b2..08cd5fd720 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.types import UserID from synapse.api.errors import ( - SynapseError, RegistrationError, InvalidCaptchaError + SynapseError, RegistrationError, InvalidCaptchaError, CodeMessageException ) from ._base import BaseHandler import synapse.util.stringutils as stringutils @@ -28,6 +28,7 @@ from synapse.http.client import CaptchaServerHttpClient import base64 import bcrypt +import json import logging logger = logging.getLogger(__name__) @@ -161,21 +162,26 @@ class RegistrationHandler(BaseHandler): def _threepid_from_creds(self, creds): # TODO: get this from the homeserver rather than creating a new one for # each request - httpCli = SimpleHttpClient(self.hs) + http_client = SimpleHttpClient(self.hs) # XXX: make this configurable! trustedIdServers = ['matrix.org:8090', 'matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) defer.returnValue(None) - data = yield httpCli.get_json( - # XXX: This should be HTTPS - "http://%s%s" % ( - creds['idServer'], - "/_matrix/identity/api/v1/3pid/getValidated3pid" - ), - {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} - ) + + data = {} + try: + data = yield http_client.get_json( + # XXX: This should be HTTPS + "http://%s%s" % ( + creds['idServer'], + "/_matrix/identity/api/v1/3pid/getValidated3pid" + ), + {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} + ) + except CodeMessageException as e: + data = json.loads(e.msg) if 'medium' in data: defer.returnValue(data) @@ -185,19 +191,23 @@ class RegistrationHandler(BaseHandler): def _bind_threepid(self, creds, mxid): yield logger.debug("binding threepid") - httpCli = SimpleHttpClient(self.hs) - data = yield httpCli.post_urlencoded_get_json( - # XXX: Change when ID servers are all HTTPS - "http://%s%s" % ( - creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" - ), - { - 'sid': creds['sid'], - 'clientSecret': creds['clientSecret'], - 'mxid': mxid, - } - ) - logger.debug("bound threepid") + http_client = SimpleHttpClient(self.hs) + data = None + try: + data = yield http_client.post_urlencoded_get_json( + # XXX: Change when ID servers are all HTTPS + "http://%s%s" % ( + creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" + ), + { + 'sid': creds['sid'], + 'clientSecret': creds['clientSecret'], + 'mxid': mxid, + } + ) + logger.debug("bound threepid") + except CodeMessageException as e: + data = json.loads(e.msg) defer.returnValue(data) @defer.inlineCallbacks diff --git a/synapse/http/client.py b/synapse/http/client.py index 198f575cfa..5f4558be47 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from synapse.api.errors import CodeMessageException from synapse.http.agent_name import AGENT_NAME from twisted.internet import defer, reactor from twisted.web.client import ( @@ -83,7 +83,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def get_json(self, uri, args={}): - """ Get's some json from the given host and path + """ Gets some json from the given host and path Args: uri (str): The URI to request, not including query parameters @@ -91,15 +91,11 @@ class SimpleHttpClient(object): None. **Note**: The value of each key is assumed to be an iterable and *not* a string. - Returns: - Deferred: Succeeds when we get *any* HTTP response. - - The result of the deferred is a tuple of `(code, response)`, - where `response` is a dict representing the decoded JSON body. + Deferred: Succeeds when we get *any* 2xx HTTP response. + Raises: + On a non-2xx HTTP response. """ - - yield if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) @@ -114,7 +110,10 @@ class SimpleHttpClient(object): body = yield readBody(response) - defer.returnValue(json.loads(body)) + if 200 <= response.code < 300: + defer.returnValue(json.loads(body)) + else: + raise CodeMessageException(response.code, body) class CaptchaServerHttpClient(SimpleHttpClient): -- cgit 1.4.1 From a1a4960baf78c1a24f76c603076c832a1947360f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 09:43:22 +0000 Subject: Impl push_bulk function --- synapse/appservice/api.py | 19 ++++++++++++++++++- synapse/handlers/appservice.py | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 74508ecddf..fbf4abc526 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -64,8 +64,25 @@ class ApplicationServiceApi(SimpleHttpClient): return logger.warning("query_alias to %s received %s", uri, e.code) + @defer.inlineCallbacks def push_bulk(self, service, events): - pass + uri = service.url + ("/transactions/%s" % + urllib.quote(str(0))) # TODO txn_ids + response = None + try: + response = yield self.put_json( + uri, + { + "events": events + }, + { + "access_token": self.hs_token + }) + if response: # just an empty json object + defer.returnValue(True) + except CodeMessageException as e: + logger.warning("push_bulk to %s received %s", uri, e.code) + defer.returnValue(False) @defer.inlineCallbacks def push(self, service, event): diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index dd860b2244..2b2761682f 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -56,7 +56,7 @@ class ApplicationServicesHandler(object): yield self.store.update_app_service(app_service) logger.info("Sending ping to %s...", app_service.url) - yield self.appservice_api.query_alias(app_service, "ping") + yield self.appservice_api.push(app_service, "pinger") def unregister(self, token): logger.info("Unregister as_token=%s", token) -- cgit 1.4.1 From 27091f146a0ebdbfe1ae7c5cd30de51515cfbebc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 10:08:12 +0000 Subject: Add hs_token column and generate a different token f.e application service. --- synapse/appservice/__init__.py | 6 ++++-- synapse/appservice/api.py | 8 ++++---- synapse/handlers/appservice.py | 9 ++++++--- synapse/rest/appservice/v1/register.py | 4 ++-- synapse/storage/appservice.py | 17 ++++++++++++----- synapse/storage/schema/application_services.sql | 1 + tests/storage/test_appservice.py | 10 ++++++---- 7 files changed, 35 insertions(+), 20 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 0c7f58574e..f7baf578f0 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -34,11 +34,13 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - def __init__(self, token, url=None, namespaces=None, txn_id=None): + def __init__(self, token, url=None, namespaces=None, hs_token=None, + txn_id=None): self.token = token self.url = url + self.hs_token = hs_token self.namespaces = self._check_namespaces(namespaces) - self.txn_id = None + self.txn_id = txn_id def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index fbf4abc526..29bb35d61b 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -30,7 +30,6 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) - self.hs_token = "_hs_token_" # TODO extract hs token @defer.inlineCallbacks def query_user(self, service, user_id): @@ -38,7 +37,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.get_json(uri, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object defer.returnValue(True) @@ -54,7 +53,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.get_json(uri, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object defer.returnValue(True) @@ -76,9 +75,10 @@ class ApplicationServiceApi(SimpleHttpClient): "events": events }, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object + # TODO: Mark txn as sent successfully defer.returnValue(True) except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2b2761682f..7b0599c71e 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -19,6 +19,7 @@ from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi +import synapse.util.stringutils as stringutils import logging @@ -53,10 +54,9 @@ class ApplicationServicesHandler(object): errcode=Codes.FORBIDDEN ) logger.info("Updating application service info...") + app_service.hs_token = self._generate_hs_token() yield self.store.update_app_service(app_service) - - logger.info("Sending ping to %s...", app_service.url) - yield self.appservice_api.push(app_service, "pinger") + defer.returnValue(app_service) def unregister(self, token): logger.info("Unregister as_token=%s", token) @@ -136,3 +136,6 @@ class ApplicationServicesHandler(object): # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + + def _generate_hs_token(self): + return stringutils.random_string(18) diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index e374d538e7..d3d5aef220 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -61,8 +61,8 @@ class RegisterRestServlet(AppServiceRestServlet): app_service = ApplicationService(as_token, as_url, namespaces) - yield self.handler.register(app_service) - hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? + app_service = yield self.handler.register(app_service) + hs_token = app_service.hs_token defer.returnValue((200, { "hs_token": hs_token diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index b64416de28..3c8bf9ad0d 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -60,6 +60,7 @@ class ApplicationServiceStore(SQLBaseStore): if service.token == token: service.url = None service.namespaces = None + service.hs_token = None def _unregister_app_service_txn(self, txn, token): # kill the url to prevent pushes @@ -100,6 +101,9 @@ class ApplicationServiceStore(SQLBaseStore): if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") + if not service.hs_token: + raise StoreError(500, "No HS token") + yield self.runInteraction( "update_app_service", self._update_app_service_txn, @@ -126,8 +130,8 @@ class ApplicationServiceStore(SQLBaseStore): return False txn.execute( - "UPDATE application_services SET url=? WHERE id=?", - (service.url, as_id,) + "UPDATE application_services SET url=?, hs_token=? WHERE id=?", + (service.url, service.hs_token, as_id,) ) # cleanup regex txn.execute( @@ -196,6 +200,7 @@ class ApplicationServiceStore(SQLBaseStore): # 'namespace': enum, # 'as_id': 0, # 'token': "something", + # 'hs_token': "otherthing", # 'id': 0 # } # ] @@ -208,6 +213,7 @@ class ApplicationServiceStore(SQLBaseStore): services[as_token] = { "url": res["url"], "token": as_token, + "hs_token": res["hs_token"], "namespaces": { ApplicationService.NS_USERS: [], ApplicationService.NS_ALIASES: [], @@ -230,8 +236,9 @@ class ApplicationServiceStore(SQLBaseStore): for service in services.values(): logger.info("Found application service: %s", service) self.cache.services.append(ApplicationService( - service["token"], - service["url"], - service["namespaces"] + token=service["token"], + url=service["url"], + namespaces=service["namespaces"], + hs_token=service["hs_token"] )) diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql index 6d245fc807..03b5a10c8a 100644 --- a/synapse/storage/schema/application_services.sql +++ b/synapse/storage/schema/application_services.sql @@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS application_services( id INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT, token TEXT, + hs_token TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 56fdda377c..b9ecfb3384 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -46,13 +46,15 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_update_and_retrieval_of_service(self): url = "https://matrix.org/appservices/foobar" + hs_token = "hstok" user_regex = ["@foobar_.*:matrix.org"] alias_regex = ["#foobar_.*:matrix.org"] room_regex = [] - service = ApplicationService(url=url, token=self.as_token, namespaces={ - ApplicationService.NS_USERS: user_regex, - ApplicationService.NS_ALIASES: alias_regex, - ApplicationService.NS_ROOMS: room_regex + service = ApplicationService( + url=url, hs_token=hs_token, token=self.as_token, namespaces={ + ApplicationService.NS_USERS: user_regex, + ApplicationService.NS_ALIASES: alias_regex, + ApplicationService.NS_ROOMS: room_regex }) yield self.store.update_app_service(service) -- cgit 1.4.1 From f0c730252fcb198ad925d6406f0f37ccee29e720 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 11:25:32 +0000 Subject: Add unknown user ID check. Use store.get_aliases_for_room(room_id) when searching for services by alias. --- synapse/handlers/appservice.py | 24 +++++++++++++++++++----- tests/handlers/test_appservice.py | 1 + 2 files changed, 20 insertions(+), 5 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 7b0599c71e..39976b9629 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,10 +15,10 @@ from twisted.internet import defer -from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi +from synapse.types import UserID import synapse.util.stringutils as stringutils import logging @@ -74,7 +74,7 @@ class ApplicationServicesHandler(object): event based on the service regex. """ # We need to know the aliases associated with this event.room_id, if any - alias_list = [] # TODO + alias_list = yield self.store.get_aliases_for_room(event.room_id) services = yield self.store.get_app_services() interested_list = [ s for s in services if ( @@ -100,7 +100,7 @@ class ApplicationServicesHandler(object): # Do we know this user exists? If not, poke the user query API for # all services which match that user regex. - unknown_user = False # TODO check + unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: user_query_services = yield self.get_services_for_event( event=event, @@ -117,7 +117,7 @@ class ApplicationServicesHandler(object): # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. - unknown_room_alias = False # TODO check + unknown_room_alias = False # TODO if unknown_room_alias: alias = "something" # TODO alias_query_services = yield self.get_services_for_event( @@ -137,5 +137,19 @@ class ApplicationServicesHandler(object): for service in services: self.appservice_api.push(service, event) + @defer.inlineCallbacks + def _is_unknown_user(self, user_id): + user = UserID.from_string(user_id) + if not self.hs.is_mine(user): + # we don't know if they are unknown or not since it isn't one of our + # users. We can't poke ASes. + defer.returnValue(False) + return + + user_info = yield self.store.get_user_by_id(user_id) + defer.returnValue(len(user_info) == 0) + + + def _generate_hs_token(self): - return stringutils.random_string(18) + return stringutils.random_string(24) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 1daa314f20..f92dc1c89a 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -46,6 +46,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): ] self.mock_store.get_app_services = Mock(return_value=services) + self.mock_store.get_user_by_id = Mock(return_value=[]) event = Mock( sender="@someone:anywhere", -- cgit 1.4.1 From b932600653a4585968a7d177b7ea7cb2ca33642a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 11:47:11 +0000 Subject: Add unknown room alias check. Call it from directory_handler.get_association --- synapse/handlers/appservice.py | 97 +++++++++++++++++++++++------------------- synapse/handlers/directory.py | 7 +++ 2 files changed, 61 insertions(+), 43 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 39976b9629..2c6d4e2815 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -62,27 +62,6 @@ class ApplicationServicesHandler(object): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) - @defer.inlineCallbacks - def get_services_for_event(self, event, restrict_to=""): - """Retrieve a list of application services interested in this event. - - Args: - event(Event): The event to check. - restrict_to(str): The namespace to restrict regex tests to. - Returns: - list: A list of services interested in this - event based on the service regex. - """ - # We need to know the aliases associated with this event.room_id, if any - alias_list = yield self.store.get_aliases_for_room(event.room_id) - services = yield self.store.get_app_services() - interested_list = [ - s for s in services if ( - s.is_interested(event, restrict_to, alias_list) - ) - ] - defer.returnValue(interested_list) - @defer.inlineCallbacks def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. @@ -94,7 +73,7 @@ class ApplicationServicesHandler(object): event(Event): The event to push out to interested services. """ # Gather interested services - services = yield self.get_services_for_event(event) + services = yield self._get_services_for_event(event) if len(services) == 0: return # no services need notifying @@ -102,7 +81,7 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: - user_query_services = yield self.get_services_for_event( + user_query_services = yield self._get_services_for_event( event=event, restrict_to=ApplicationService.NS_USERS ) @@ -115,28 +94,62 @@ class ApplicationServicesHandler(object): # the user exists now,so don't query more ASes. break - # Do we know this room alias exists? If not, poke the room alias query - # API for all services which match that room alias regex. - unknown_room_alias = False # TODO - if unknown_room_alias: - alias = "something" # TODO - alias_query_services = yield self.get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_ALIASES - ) - for alias_service in alias_query_services: - # this needs to block XXX: Need to feed response back to caller - is_known_alias = yield self.appservice_api.query_alias( - alias_service, alias - ) - if is_known_alias: - # the alias exists now so don't query more ASes. - break - # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + + @defer.inlineCallbacks + def query_room_alias_exists(self, room_alias): + """Check if an application service knows this room alias exists. + + Args: + room_alias(str): The room alias to query. + Returns: + namedtuple: with keys "room_id" and "servers" or None if no + association can be found. + """ + alias_query_services = yield self._get_services_for_event( + event=None, + restrict_to=ApplicationService.NS_ALIASES, + alias_list=[room_alias] + ) + for alias_service in alias_query_services: + is_known_alias = yield self.appservice_api.query_alias( + alias_service, room_alias + ) + if is_known_alias: + # the alias exists now so don't query more ASes. + result = yield self.store.get_association_from_room_alias( + room_alias + ) + defer.returnValue(result) + + @defer.inlineCallbacks + def _get_services_for_event(self, event, restrict_to="", alias_list=None): + """Retrieve a list of application services interested in this event. + + Args: + event(Event): The event to check. Can be None if alias_list is not. + restrict_to(str): The namespace to restrict regex tests to. + alias_list: A list of aliases to get services for. If None, this + list is obtained from the database. + Returns: + list: A list of services interested in this + event based on the service regex. + """ + # We need to know the aliases associated with this event.room_id, if any + if not alias_list: + alias_list = yield self.store.get_aliases_for_room(event.room_id) + + services = yield self.store.get_app_services() + interested_list = [ + s for s in services if ( + s.is_interested(event, restrict_to, alias_list) + ) + ] + defer.returnValue(interested_list) + @defer.inlineCallbacks def _is_unknown_user(self, user_id): user = UserID.from_string(user_id) @@ -149,7 +162,5 @@ class ApplicationServicesHandler(object): user_info = yield self.store.get_user_by_id(user_id) defer.returnValue(len(user_info) == 0) - - def _generate_hs_token(self): return stringutils.random_string(24) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 58e9a91562..000bf5793c 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -84,6 +84,13 @@ class DirectoryHandler(BaseHandler): if result: room_id = result.room_id servers = result.servers + else: + # Query AS to see if it exists + as_handler = self.hs.get_handlers().appservice_handler + result = yield as_handler.query_room_alias_exists(room_alias) + if result: + room_id = result.room_id + servers = result.servers else: try: result = yield self.federation.make_query( -- cgit 1.4.1 From 26a041541baed887dc069c3667a86ddef81802bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Feb 2015 13:17:05 +0000 Subject: SYN-202: Log as WARN the 404 'Presence information not visible' errors instead of as ERROR since they were spamming the logs --- synapse/handlers/message.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6fbd2af4ab..3f51f38f18 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import RoomError +from synapse.api.errors import RoomError, SynapseError from synapse.streams.config import PaginationConfig from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator @@ -372,10 +372,17 @@ class MessageHandler(BaseHandler): as_event=True, ) presence.append(member_presence) - except Exception: - logger.exception( - "Failed to get member presence of %r", m.user_id - ) + except SynapseError as e: + if e.code == 404: + # FIXME: We are doing this as a warn since this gets hit a + # lot and spams the logs. Why is this happening? + logger.warn( + "Failed to get member presence of %r", m.user_id + ) + else: + logger.exception( + "Failed to get member presence of %r", m.user_id + ) time_now = self.clock.time_msec() -- cgit 1.4.1 From 51d63ac329a2613c0a7195e6183d70f789b7d823 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 13:19:46 +0000 Subject: Glue AS work to general event notifications. Add more exception handling when poking ASes. --- synapse/appservice/__init__.py | 2 +- synapse/appservice/api.py | 11 ++++++++++- synapse/handlers/appservice.py | 39 +++++++++++++++++++++++++-------------- synapse/notifier.py | 6 ++++++ 4 files changed, 42 insertions(+), 16 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f7baf578f0..d9ca856c8b 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -62,7 +62,7 @@ class ApplicationService(object): def _matches_regex(self, test_string, namespace_key): if not isinstance(test_string, basestring): - logger.warning( + logger.error( "Expected a string to test regex against, but got %s", test_string ) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 29bb35d61b..d96caf7f58 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -46,6 +46,9 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) return logger.warning("query_user to %s received %s", uri, e.code) + except Exception as ex: + logger.warning("query_user to %s threw exception %s", uri, ex) + defer.returnValue(False) @defer.inlineCallbacks def query_alias(self, service, alias): @@ -62,6 +65,10 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) return logger.warning("query_alias to %s received %s", uri, e.code) + except Exception as ex: + logger.warning("query_alias to %s threw exception %s", uri, ex) + defer.returnValue(False) + @defer.inlineCallbacks def push_bulk(self, service, events): @@ -82,7 +89,9 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(True) except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) - defer.returnValue(False) + except Exception as ex: + logger.warning("push_bulk to %s threw exception %s", uri, ex) + defer.returnValue(False) @defer.inlineCallbacks def push(self, service, event): diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2c6d4e2815..ef94215133 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -23,7 +23,6 @@ import synapse.util.stringutils as stringutils import logging - logger = logging.getLogger(__name__) @@ -58,6 +57,7 @@ class ApplicationServicesHandler(object): yield self.store.update_app_service(app_service) defer.returnValue(app_service) + @defer.inlineCallbacks def unregister(self, token): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) @@ -81,34 +81,45 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: - user_query_services = yield self._get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_USERS - ) - for user_service in user_query_services: - # this needs to block XXX: Need to feed response back to caller - is_known_user = yield self.appservice_api.query_user( - user_service, event.sender - ) - if is_known_user: - # the user exists now,so don't query more ASes. - break + yield self.query_user_exists(event) # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + @defer.inlineCallbacks + def query_user_exists(self, event): + """Check if an application services knows this event.sender exists. + + Args: + event: An event sent by the user to query + Returns: + True if this user exists. + """ + # TODO Would be nice for this to accept a user ID instead of an event. + user_query_services = yield self._get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_USERS + ) + for user_service in user_query_services: + is_known_user = yield self.appservice_api.query_user( + user_service, event.sender + ) + if is_known_user: + defer.returnValue(True) + defer.returnValue(False) @defer.inlineCallbacks def query_room_alias_exists(self, room_alias): """Check if an application service knows this room alias exists. Args: - room_alias(str): The room alias to query. + room_alias(RoomAlias): The room alias to query. Returns: namedtuple: with keys "room_id" and "servers" or None if no association can be found. """ + room_alias = room_alias.to_string() alias_query_services = yield self._get_services_for_event( event=None, restrict_to=ApplicationService.NS_ALIASES, diff --git a/synapse/notifier.py b/synapse/notifier.py index e3b6ead620..c7f75ab801 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -99,6 +99,12 @@ class Notifier(object): `extra_users` param. """ yield run_on_reactor() + + # poke any interested application service. + self.hs.get_handlers().appservice_handler.notify_interested_services( + event + ) + room_id = event.room_id room_source = self.event_sources.sources["room"] -- cgit 1.4.1 From e1515c3e91f2117adc3976b5e606728560ce9e96 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Feb 2015 13:43:28 +0000 Subject: Pass through list of room hosts from room alias query to federation so that it can retry against different room hosts --- synapse/federation/federation_client.py | 5 ++++- synapse/handlers/federation.py | 20 +++++++++++++------- synapse/handlers/room.py | 12 +++++------- 3 files changed, 22 insertions(+), 15 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index eb36ec040b..9923b3fc0d 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -264,7 +264,9 @@ class FederationClient(FederationBase): logger.debug("Got response to make_join: %s", pdu_dict) - defer.returnValue(self.event_from_pdu_json(pdu_dict)) + defer.returnValue( + (destination, self.event_from_pdu_json(pdu_dict)) + ) break except CodeMessageException: raise @@ -313,6 +315,7 @@ class FederationClient(FederationBase): defer.returnValue({ "state": signed_state, "auth_chain": signed_auth, + "origin": destination, }) except CodeMessageException: raise diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 04a4689483..aba266c2bc 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -273,7 +273,7 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks - def do_invite_join(self, target_host, room_id, joinee, content, snapshot): + def do_invite_join(self, target_hosts, room_id, joinee, content, snapshot): """ Attempts to join the `joinee` to the room `room_id` via the server `target_host`. @@ -287,8 +287,8 @@ class FederationHandler(BaseHandler): """ logger.debug("Joining %s to %s", joinee, room_id) - pdu = yield self.replication_layer.make_join( - [target_host], + origin, pdu = yield self.replication_layer.make_join( + target_hosts, room_id, joinee ) @@ -330,11 +330,17 @@ class FederationHandler(BaseHandler): new_event = builder.build() + # Try the host we successfully got a response to /make_join/ + # request first. + target_hosts.remove(origin) + target_hosts.insert(0, origin) + ret = yield self.replication_layer.send_join( - [target_host], + target_hosts, new_event ) + origin = ret["origin"] state = ret["state"] auth_chain = ret["auth_chain"] auth_chain.sort(key=lambda e: e.depth) @@ -371,7 +377,7 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } yield self._handle_new_event( - target_host, e, auth_events=auth + origin, e, auth_events=auth ) except: logger.exception( @@ -391,7 +397,7 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } yield self._handle_new_event( - target_host, e, auth_events=auth + origin, e, auth_events=auth ) except: logger.exception( @@ -406,7 +412,7 @@ class FederationHandler(BaseHandler): } yield self._handle_new_event( - target_host, + origin, new_event, state=state, current_state=state, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 23821d321f..0369b907a5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -389,8 +389,6 @@ class RoomMemberHandler(BaseHandler): if not hosts: raise SynapseError(404, "No known servers") - host = hosts[0] - # If event doesn't include a display name, add one. yield self.distributor.fire( "collect_presencelike_data", joinee, content @@ -407,12 +405,12 @@ class RoomMemberHandler(BaseHandler): }) event, context = yield self._create_new_client_event(builder) - yield self._do_join(event, context, room_host=host, do_auth=True) + yield self._do_join(event, context, room_hosts=hosts, do_auth=True) defer.returnValue({"room_id": room_id}) @defer.inlineCallbacks - def _do_join(self, event, context, room_host=None, do_auth=True): + def _do_join(self, event, context, room_hosts=None, do_auth=True): joinee = UserID.from_string(event.state_key) # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id @@ -441,7 +439,7 @@ class RoomMemberHandler(BaseHandler): if is_host_in_room: should_do_dance = False - elif room_host: # TODO: Shouldn't this be remote_room_host? + elif room_hosts: # TODO: Shouldn't this be remote_room_host? should_do_dance = True else: # TODO(markjh): get prev_state from snapshot @@ -453,7 +451,7 @@ class RoomMemberHandler(BaseHandler): inviter = UserID.from_string(prev_state.user_id) should_do_dance = not self.hs.is_mine(inviter) - room_host = inviter.domain + room_hosts = [inviter.domain] else: # return the same error as join_room_alias does raise SynapseError(404, "No known servers") @@ -461,7 +459,7 @@ class RoomMemberHandler(BaseHandler): if should_do_dance: handler = self.hs.get_handlers().federation_handler yield handler.do_invite_join( - room_host, + room_hosts, room_id, event.user_id, event.get_dict()["content"], # FIXME To get a non-frozen dict -- cgit 1.4.1 From c71456117dc70dfe0bfa15e3f655a3ac1dfc66ee Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 14:17:08 +0000 Subject: Fix user query checks. HS>AS pushing now works. --- synapse/appservice/__init__.py | 21 ++++++++++++-------- synapse/handlers/appservice.py | 44 +++++++++++++++++++++++++++++------------- 2 files changed, 44 insertions(+), 21 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index d9ca856c8b..46d46a5a48 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -75,27 +75,23 @@ class ApplicationService(object): def _matches_user(self, event): if (hasattr(event, "sender") and - self._matches_regex( - event.sender, ApplicationService.NS_USERS)): + self.is_interested_in_user(event.sender)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member and hasattr(event, "state_key") - and self._matches_regex( - event.state_key, ApplicationService.NS_USERS)): + and self.is_interested_in_user(event.state_key)): return True return False def _matches_room_id(self, event): if hasattr(event, "room_id"): - return self._matches_regex( - event.room_id, ApplicationService.NS_ROOMS - ) + return self.is_interested_in_room(event.room_id) return False def _matches_aliases(self, event, alias_list): for alias in alias_list: - if self._matches_regex(alias, ApplicationService.NS_ALIASES): + if self.is_interested_in_alias(alias): return True return False @@ -128,5 +124,14 @@ class ApplicationService(object): elif restrict_to == ApplicationService.NS_USERS: return self._matches_user(event) + def is_interested_in_user(self, user_id): + return self._matches_regex(user_id, ApplicationService.NS_USERS) + + def is_interested_in_alias(self, alias): + return self._matches_regex(alias, ApplicationService.NS_ALIASES) + + def is_interested_in_room(self, room_id): + return self._matches_regex(room_id, ApplicationService.NS_ROOMS) + def __str__(self): return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index ef94215133..8d0cdd528c 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,6 +15,7 @@ from twisted.internet import defer +from synapse.api.constants import EventTypes from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi @@ -78,32 +79,31 @@ class ApplicationServicesHandler(object): return # no services need notifying # Do we know this user exists? If not, poke the user query API for - # all services which match that user regex. - unknown_user = yield self._is_unknown_user(event.sender) - if unknown_user: - yield self.query_user_exists(event) + # all services which match that user regex. This needs to block as these + # user queries need to be made BEFORE pushing the event. + yield self._check_user_exists(event.sender) + if event.type == EventTypes.Member: + yield self._check_user_exists(event.state_key) # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) @defer.inlineCallbacks - def query_user_exists(self, event): - """Check if an application services knows this event.sender exists. + def query_user_exists(self, user_id): + """Check if any application service knows this user_id exists. Args: - event: An event sent by the user to query + user_id(str): The user to query if they exist on any AS. Returns: - True if this user exists. + True if this user exists on at least one application service. """ - # TODO Would be nice for this to accept a user ID instead of an event. - user_query_services = yield self._get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_USERS + user_query_services = yield self._get_services_for_user( + user_id=user_id ) for user_service in user_query_services: is_known_user = yield self.appservice_api.query_user( - user_service, event.sender + user_service, user_id ) if is_known_user: defer.returnValue(True) @@ -161,6 +161,16 @@ class ApplicationServicesHandler(object): ] defer.returnValue(interested_list) + @defer.inlineCallbacks + def _get_services_for_user(self, user_id): + services = yield self.store.get_app_services() + interested_list = [ + s for s in services if ( + s.is_interested_in_user(user_id) + ) + ] + defer.returnValue(interested_list) + @defer.inlineCallbacks def _is_unknown_user(self, user_id): user = UserID.from_string(user_id) @@ -173,5 +183,13 @@ class ApplicationServicesHandler(object): user_info = yield self.store.get_user_by_id(user_id) defer.returnValue(len(user_info) == 0) + @defer.inlineCallbacks + def _check_user_exists(self, user_id): + unknown_user = yield self._is_unknown_user(user_id) + if unknown_user: + exists = yield self.query_user_exists(user_id) + defer.returnValue(exists) + defer.returnValue(True) + def _generate_hs_token(self): return stringutils.random_string(24) -- cgit 1.4.1 From e9484d6a9514d5c9aea3839bcc3fb25c9cb88cb0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 16:29:56 +0000 Subject: Prevent aliases in AS namespaces being created/deleted by users. Check with ASes when queried for room aliases via federation. --- synapse/handlers/directory.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 24ea3573d3..842f075fe0 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -49,6 +49,12 @@ class DirectoryHandler(BaseHandler): # TODO(erikj): Check if there is a current association. + is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) + if is_claimed: + raise SynapseError( + 400, "This alias is reserved by an application service." + ) + if not servers: servers = yield self.store.get_joined_hosts_for_room(room_id) @@ -68,6 +74,12 @@ class DirectoryHandler(BaseHandler): if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") + is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) + if is_claimed: + raise SynapseError( + 400, "This alias is reserved by an application service." + ) + room_id = yield self.store.delete_room_alias(room_alias) if room_id: @@ -77,20 +89,13 @@ class DirectoryHandler(BaseHandler): def get_association(self, room_alias): room_id = None if self.hs.is_mine(room_alias): - result = yield self.store.get_association_from_room_alias( + result = yield self.get_association_from_room_alias( room_alias ) if result: room_id = result.room_id servers = result.servers - else: - # Query AS to see if it exists - as_handler = self.hs.get_handlers().appservice_handler - result = yield as_handler.query_room_alias_exists(room_alias) - if result: - room_id = result.room_id - servers = result.servers else: try: result = yield self.federation.make_query( @@ -145,7 +150,7 @@ class DirectoryHandler(BaseHandler): 400, "Room Alias is not hosted on this Home Server" ) - result = yield self.store.get_association_from_room_alias( + result = yield self.get_association_from_room_alias( room_alias ) @@ -173,3 +178,22 @@ class DirectoryHandler(BaseHandler): "sender": user_id, "content": {"aliases": aliases}, }, ratelimit=False) + + @defer.inlineCallbacks + def get_association_from_room_alias(self, room_alias): + result = yield self.store.get_association_from_room_alias( + room_alias + ) + if not result: + # Query AS to see if it exists + as_handler = self.hs.get_handlers().appservice_handler + result = yield as_handler.query_room_alias_exists(room_alias) + defer.returnValue(result) + + @defer.inlineCallbacks + def is_alias_exclusive_to_appservices(self, alias): + services = yield self.store.get_app_services() + interested_services = [ + s for s in services if s.is_interested_in_alias(alias.to_string()) + ] + defer.returnValue(len(interested_services) > 0) -- cgit 1.4.1 From cab4c730885dcb5c95a50d425d6b6f655154a173 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 16:46:56 +0000 Subject: Prevent user IDs in AS namespaces being created/deleted by humans. --- synapse/handlers/register.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'synapse/handlers') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 08cd5fd720..b6e19d498c 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -65,6 +65,8 @@ class RegistrationHandler(BaseHandler): user = UserID(localpart, self.hs.hostname) user_id = user.to_string() + yield self.check_user_id_is_valid(user_id) + token = self._generate_token(user_id) yield self.store.register( user_id=user_id, @@ -83,6 +85,7 @@ class RegistrationHandler(BaseHandler): localpart = self._generate_user_id() user = UserID(localpart, self.hs.hostname) user_id = user.to_string() + yield self.check_user_id_is_valid(user_id) token = self._generate_token(user_id) yield self.store.register( @@ -148,6 +151,19 @@ class RegistrationHandler(BaseHandler): # XXX: This should be a deferred list, shouldn't it? yield self._bind_threepid(c, user_id) + @defer.inlineCallbacks + def check_user_id_is_valid(self, user_id): + # valid user IDs must not clash with any user ID namespaces claimed by + # application services. + services = yield self.store.get_app_services() + interested_services = [ + s for s in services if s.is_interested_in_user(user_id) + ] + if len(interested_services) > 0: + raise SynapseError( + 400, "This user ID is reserved by an application service." + ) + def _generate_token(self, user_id): # urlsafe variant uses _ and - so use . as the separator and replace # all =s with .s so http clients don't quote =s when it is used as -- cgit 1.4.1 From 11e6b3d18b5ae95857e01884960a01b9ea4d307d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 17:04:59 +0000 Subject: Dependency inject ApplicationServiceApi when creating ApplicationServicesHandler. --- synapse/handlers/__init__.py | 5 ++++- synapse/handlers/appservice.py | 5 ++--- tests/handlers/test_appservice.py | 10 +++------- 3 files changed, 9 insertions(+), 11 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index b31518bf62..8d345bf936 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.appservice.api import ApplicationServiceApi from .register import RegistrationHandler from .room import ( RoomCreationHandler, RoomMemberHandler, RoomListHandler @@ -53,5 +54,7 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) - self.appservice_handler = ApplicationServicesHandler(hs) + self.appservice_handler = ApplicationServicesHandler( + hs, ApplicationServiceApi(hs) + ) self.sync_handler = SyncHandler(hs) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8d0cdd528c..fa810b9a98 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -18,7 +18,6 @@ from twisted.internet import defer from synapse.api.constants import EventTypes from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService -from synapse.appservice.api import ApplicationServiceApi from synapse.types import UserID import synapse.util.stringutils as stringutils @@ -32,10 +31,10 @@ logger = logging.getLogger(__name__) # easier. class ApplicationServicesHandler(object): - def __init__(self, hs): + def __init__(self, hs, appservice_api): self.store = hs.get_datastore() self.hs = hs - self.appservice_api = ApplicationServiceApi(hs) + self.appservice_api = appservice_api @defer.inlineCallbacks def register(self, app_service): diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index e16e511587..a2c541317c 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -29,13 +29,9 @@ class AppServiceHandlerTestCase(unittest.TestCase): self.mock_as_api = Mock() hs = Mock() hs.get_datastore = Mock(return_value=self.mock_store) - self.handler = ApplicationServicesHandler(hs) # thing being tested - - # FIXME Would be nice to DI this rather than monkey patch:( - if not hasattr(self.handler, "appservice_api"): - # someone probably updated the handler but not the tests. Fail fast. - raise Exception("Test expected handler.appservice_api to exist.") - self.handler.appservice_api = self.mock_as_api + self.handler = ApplicationServicesHandler( + hs, self.mock_as_api + ) @defer.inlineCallbacks def test_notify_interested_services(self): -- cgit 1.4.1 From 0227618d3c3bea6c85a922d5605f526719573121 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 17:29:27 +0000 Subject: Add m.login.application_service registration procedure. This allows known application services to register any user ID under their own user namespace(s). --- synapse/api/constants.py | 1 + synapse/handlers/register.py | 20 ++++++++++++++++++++ synapse/rest/client/v1/register.py | 24 +++++++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 0d3fc629af..420f963d91 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -59,6 +59,7 @@ class LoginType(object): EMAIL_URL = u"m.login.email.url" EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" + APPLICATION_SERVICE = u"m.login.application_service" class EventTypes(object): diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index b6e19d498c..60821edb05 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -105,6 +105,26 @@ class RegistrationHandler(BaseHandler): defer.returnValue((user_id, token)) + @defer.inlineCallbacks + def appservice_register(self, user_localpart, as_token): + user = UserID(user_localpart, self.hs.hostname) + user_id = user.to_string() + service = yield self.store.get_app_service_by_token(as_token) + if not service: + raise SynapseError(403, "Invalid application service token.") + if not service.is_interested_in_user(user_id): + raise SynapseError( + 400, "Invalid user localpart for this application service." + ) + token = self._generate_token(user_id) + yield self.store.register( + user_id=user_id, + token=token, + password_hash="" + ) + self.distributor.fire("registered_user", user) + defer.returnValue((user_id, token)) + @defer.inlineCallbacks def check_recaptcha(self, ip, private_key, challenge, response): """Checks a recaptcha is correct.""" diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index c0423c2d45..1ab32b53ea 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -110,7 +110,8 @@ class RegisterRestServlet(ClientV1RestServlet): stages = { LoginType.RECAPTCHA: self._do_recaptcha, LoginType.PASSWORD: self._do_password, - LoginType.EMAIL_IDENTITY: self._do_email_identity + LoginType.EMAIL_IDENTITY: self._do_email_identity, + LoginType.APPLICATION_SERVICE: self._do_app_service } session_info = self._get_session_info(request, session) @@ -276,6 +277,27 @@ class RegisterRestServlet(ClientV1RestServlet): self._remove_session(session) defer.returnValue(result) + @defer.inlineCallbacks + def _do_app_service(self, request, register_json, session): + if "access_token" not in request.args: + raise SynapseError(400, "Expected application service token.") + if "user" not in register_json: + raise SynapseError(400, "Expected 'user' key.") + + as_token = request.args["access_token"][0] + user_localpart = register_json["user"].encode("utf-8") + + handler = self.handlers.registration_handler + (user_id, token) = yield handler.appservice_register( + user_localpart, as_token + ) + self._remove_session(session) + defer.returnValue({ + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + }) + def _parse_json(request): try: -- cgit 1.4.1 From 3737329d9b40b54bc4205c5f5e7e0946c5e51614 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 10:53:18 +0000 Subject: Handle the fact the list.remove raises if element doesn't exist --- synapse/handlers/federation.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index aba266c2bc..9d0ce9aa50 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -332,8 +332,11 @@ class FederationHandler(BaseHandler): # Try the host we successfully got a response to /make_join/ # request first. - target_hosts.remove(origin) - target_hosts.insert(0, origin) + try: + target_hosts.remove(origin) + target_hosts.insert(0, origin) + except ValueError: + pass ret = yield self.replication_layer.send_join( target_hosts, @@ -521,7 +524,7 @@ class FederationHandler(BaseHandler): "Failed to get destination from event %s", s.event_id ) - destinations.remove(origin) + destinations.discard(origin) logger.debug( "on_send_join_request: Sending event: %s, signatures: %s", @@ -1013,7 +1016,10 @@ class FederationHandler(BaseHandler): for e in missing_remotes: for e_id, _ in e.auth_events: if e_id in missing_remote_ids: - base_remote_rejected.remove(e) + try: + base_remote_rejected.remove(e) + except ValueError: + pass reason_map = {} -- cgit 1.4.1 From e426df8e106aef0b213928afb6189569474ac5d9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 10:57:14 +0000 Subject: Grant ASes the ability to create alias in their own namespace. Add a new errcode type M_EXCLUSIVE when users try to create aliases inside AS namespaces, and when ASes try to create aliases outside their own namespace. --- synapse/api/auth.py | 12 +++++++++++ synapse/api/errors.py | 3 ++- synapse/handlers/directory.py | 43 ++++++++++++++++++++++++++----------- synapse/rest/client/v1/directory.py | 29 +++++++++++++++---------- 4 files changed, 63 insertions(+), 24 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 4f116184c9..ea8c461729 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -380,6 +380,18 @@ class Auth(object): raise AuthError(403, "Unrecognised access token.", errcode=Codes.UNKNOWN_TOKEN) + @defer.inlineCallbacks + def get_appservice_by_req(self, request): + try: + token = request.args["access_token"][0] + service = yield self.store.get_app_service_by_token(token) + if not service: + raise AuthError(403, "Unrecognised access token.", + errcode=Codes.UNKNOWN_TOKEN) + defer.returnValue(service) + except KeyError: + raise AuthError(403, "Missing access token.") + def is_server_admin(self, user): return self.store.is_server_admin(user) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 5041828f18..eddd889778 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -36,7 +36,8 @@ class Codes(object): CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" CAPTCHA_INVALID = "M_CAPTCHA_INVALID" MISSING_PARAM = "M_MISSING_PARAM", - TOO_LARGE = "M_TOO_LARGE" + TOO_LARGE = "M_TOO_LARGE", + EXCLUSIVE = "M_EXCLUSIVE" class CodeMessageException(RuntimeError): diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 842f075fe0..4c15e57fa6 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -37,24 +37,15 @@ class DirectoryHandler(BaseHandler): ) @defer.inlineCallbacks - def create_association(self, user_id, room_alias, room_id, servers=None): - - # TODO(erikj): Do auth. + def _create_association(self, room_alias, room_id, servers=None): + # general association creation for both human users and app services if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") # TODO(erikj): Change this. # TODO(erikj): Add transactions. - # TODO(erikj): Check if there is a current association. - - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: - raise SynapseError( - 400, "This alias is reserved by an application service." - ) - if not servers: servers = yield self.store.get_joined_hosts_for_room(room_id) @@ -67,6 +58,33 @@ class DirectoryHandler(BaseHandler): servers ) + + @defer.inlineCallbacks + def create_association(self, user_id, room_alias, room_id, servers=None): + # association creation for human users + # TODO(erikj): Do user auth. + + is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) + if is_claimed: + raise SynapseError( + 400, "This alias is reserved by an application service.", + errcode=Codes.EXCLUSIVE + ) + yield self._create_association(room_alias, room_id, servers) + + + @defer.inlineCallbacks + def create_appservice_association(self, service, room_alias, room_id, + servers=None): + if not service.is_interested_in_alias(room_alias.to_string()): + raise SynapseError( + 400, "This application service has not reserved" + " this kind of alias.", errcode=Codes.EXCLUSIVE + ) + + # association creation for app services + yield self._create_association(room_alias, room_id, servers) + @defer.inlineCallbacks def delete_association(self, user_id, room_alias): # TODO Check if server admin @@ -77,7 +95,8 @@ class DirectoryHandler(BaseHandler): is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) if is_claimed: raise SynapseError( - 400, "This alias is reserved by an application service." + 400, "This alias is reserved by an application service.", + errcode=Codes.EXCLUSIVE ) room_id = yield self.store.delete_room_alias(room_alias) diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 8f65efec5f..f7e910bb40 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -45,8 +45,6 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - user, client = yield self.auth.get_user_by_req(request) - content = _parse_json(request) if not "room_id" in content: raise SynapseError(400, "Missing room_id key", @@ -70,16 +68,25 @@ class ClientDirectoryServer(ClientV1RestServlet): dir_handler = self.handlers.directory_handler try: - user_id = user.to_string() - yield dir_handler.create_association( - user_id, room_alias, room_id, servers + # try to auth as a user + user, client = yield self.auth.get_user_by_req(request) + try: + user_id = user.to_string() + yield dir_handler.create_association( + user_id, room_alias, room_id, servers + ) + yield dir_handler.send_room_alias_update_event(user_id, room_id) + except SynapseError as e: + raise e + except: + logger.exception("Failed to create association") + raise + except AuthError: + # try to auth as an application service + service = yield self.auth.get_appservice_by_req(request) + yield dir_handler.create_appservice_association( + service, room_alias, room_id, servers ) - yield dir_handler.send_room_alias_update_event(user_id, room_id) - except SynapseError as e: - raise e - except: - logger.exception("Failed to create association") - raise defer.returnValue((200, {})) -- cgit 1.4.1 From c3ae8def755a043283e945e6653970599d227a43 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 11:32:07 +0000 Subject: Grant ASes the ability to delete aliases in their own namespace. --- synapse/handlers/directory.py | 28 +++++++++++++++++++++++----- synapse/rest/client/v1/directory.py | 31 +++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 7 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 4c15e57fa6..c3c95996e7 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -87,10 +87,9 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def delete_association(self, user_id, room_alias): - # TODO Check if server admin + # association deletion for human users - if not self.hs.is_mine(room_alias): - raise SynapseError(400, "Room alias must be local") + # TODO Check if server admin is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) if is_claimed: @@ -99,10 +98,29 @@ class DirectoryHandler(BaseHandler): errcode=Codes.EXCLUSIVE ) + yield self._delete_association(room_alias) + + @defer.inlineCallbacks + def delete_appservice_association(self, service, room_alias): + if not service.is_interested_in_alias(room_alias.to_string()): + raise SynapseError( + 400, + "This application service has not reserved this kind of alias", + errcode=Codes.EXCLUSIVE + ) + yield self._delete_association(room_alias) + + @defer.inlineCallbacks + def _delete_association(self, room_alias): + if not self.hs.is_mine(room_alias): + raise SynapseError(400, "Room alias must be local") + room_id = yield self.store.delete_room_alias(room_alias) - if room_id: - yield self._update_room_alias_events(user_id, room_id) + # TODO - Looks like _update_room_alias_event has never been implemented + # if room_id: + # yield self._update_room_alias_events(user_id, room_id) + @defer.inlineCallbacks def get_association(self, room_alias): diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index f7e910bb40..8e83548bbf 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -87,24 +87,51 @@ class ClientDirectoryServer(ClientV1RestServlet): yield dir_handler.create_appservice_association( service, room_alias, room_id, servers ) + logger.info( + "Application service at %s created alias %s pointing to %s", + service.url, + room_alias.to_string(), + room_id + ) defer.returnValue((200, {})) @defer.inlineCallbacks def on_DELETE(self, request, room_alias): + dir_handler = self.handlers.directory_handler + + try: + service = yield self.auth.get_appservice_by_req(request) + room_alias = RoomAlias.from_string(room_alias) + yield dir_handler.delete_appservice_association( + service, room_alias + ) + logger.info( + "Application service at %s deleted alias %s", + service.url, + room_alias.to_string() + ) + defer.returnValue((200, {})) + except AuthError: + # fallback to default user behaviour if they aren't an AS + pass + user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: raise AuthError(403, "You need to be a server admin") - dir_handler = self.handlers.directory_handler - room_alias = RoomAlias.from_string(room_alias) yield dir_handler.delete_association( user.to_string(), room_alias ) + logger.info( + "User %s deleted alias %s", + user.to_string(), + room_alias.to_string() + ) defer.returnValue((200, {})) -- cgit 1.4.1 From 0995810273ddbc7e5e89b56ad3768fa8f0583bc4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 11:45:19 +0000 Subject: Pyflakes: unused variable. --- synapse/handlers/directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index c3c95996e7..22c6391a15 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -115,7 +115,7 @@ class DirectoryHandler(BaseHandler): if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") - room_id = yield self.store.delete_room_alias(room_alias) + yield self.store.delete_room_alias(room_alias) # TODO - Looks like _update_room_alias_event has never been implemented # if room_id: -- cgit 1.4.1 From c78b5fb1f16957f5398517fa5a99dcb370992b26 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 13:52:16 +0000 Subject: Make seen_ids a set --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9d0ce9aa50..1b709e5e24 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -130,7 +130,9 @@ class FederationHandler(BaseHandler): if auth_chain: event_ids |= {e.event_id for e in auth_chain} - seen_ids = (yield self.store.have_events(event_ids)).keys() + seen_ids = set( + (yield self.store.have_events(event_ids)).keys() + ) if state and auth_chain is not None: # If we have any state or auth_chain given to us by the replication -- cgit 1.4.1 From e890ce223c0c7798efc9961ca1e4c4870cfc7e2f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 14:16:50 +0000 Subject: Don't query auth if the only difference is events that were rejected due to auth. --- synapse/handlers/federation.py | 128 +++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 56 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1b709e5e24..b13b7c7701 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -791,12 +791,12 @@ class FederationHandler(BaseHandler): @log_function def do_auth(self, origin, event, context, auth_events): # Check if we have all the auth events. - res = yield self.store.have_events( + have_events = yield self.store.have_events( [e_id for e_id, _ in event.auth_events] ) event_auth_events = set(e_id for e_id, _ in event.auth_events) - seen_events = set(res.keys()) + seen_events = set(have_events.keys()) missing_auth = event_auth_events - seen_events @@ -839,6 +839,11 @@ class FederationHandler(BaseHandler): auth_events[(e.type, e.state_key)] = e except AuthError: pass + + have_events = yield self.store.have_events( + [e_id for e_id, _ in event.auth_events] + ) + seen_events = set(have_events.keys()) except: # FIXME: logger.exception("Failed to get auth chain") @@ -852,64 +857,75 @@ class FederationHandler(BaseHandler): # Do auth conflict res. logger.debug("Different auth: %s", different_auth) - # 1. Get what we think is the auth chain. - auth_ids = self.auth.compute_auth_events( - event, context.current_state - ) - local_auth_chain = yield self.store.get_auth_chain(auth_ids) - - try: - # 2. Get remote difference. - result = yield self.replication_layer.query_auth( - origin, - event.room_id, - event.event_id, - local_auth_chain, - ) - - seen_remotes = yield self.store.have_events( - [e.event_id for e in result["auth_chain"]] + # Only do auth resolution if we have something new to say. + # We can't rove an auth failure. + do_resolution = False + for e_id in different_auth: + if e_id in have_events: + if have_events[e_id] != RejectedReason.AUTH_ERROR: + do_resolution = True + break + + if do_resolution: + # 1. Get what we think is the auth chain. + auth_ids = self.auth.compute_auth_events( + event, context.current_state ) + local_auth_chain = yield self.store.get_auth_chain(auth_ids) - # 3. Process any remote auth chain events we haven't seen. - for ev in result["auth_chain"]: - if ev.event_id in seen_remotes.keys(): - continue - - if ev.event_id == event.event_id: - continue - - try: - auth_ids = [e_id for e_id, _ in ev.auth_events] - auth = { - (e.type, e.state_key): e for e in result["auth_chain"] - if e.event_id in auth_ids - } - ev.internal_metadata.outlier = True - - logger.debug( - "do_auth %s different_auth: %s", - event.event_id, e.event_id - ) + try: + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) - yield self._handle_new_event( - origin, ev, auth_events=auth - ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) - if ev.event_id in event_auth_events: - auth_events[(ev.type, ev.state_key)] = ev - except AuthError: - pass + # 3. Process any remote auth chain events we haven't seen. + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue + + if ev.event_id == event.event_id: + continue + + try: + auth_ids = [e_id for e_id, _ in ev.auth_events] + auth = { + (e.type, e.state_key): e + for e in result["auth_chain"] + if e.event_id in auth_ids + } + ev.internal_metadata.outlier = True + + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) + + yield self._handle_new_event( + origin, ev, auth_events=auth + ) + + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev + except AuthError: + pass - except: - # FIXME: - logger.exception("Failed to query auth chain") + except: + # FIXME: + logger.exception("Failed to query auth chain") - # 4. Look at rejects and their proofs. - # TODO. + # 4. Look at rejects and their proofs. + # TODO. - context.current_state.update(auth_events) - context.state_group = None + context.current_state.update(auth_events) + context.state_group = None try: self.auth.check(event, auth_events=auth_events) @@ -1028,9 +1044,9 @@ class FederationHandler(BaseHandler): for e in base_remote_rejected: reason = yield self.store.get_rejection_reason(e.event_id) if reason is None: - # FIXME: ERRR?! - logger.warn("Could not find reason for %s", e.event_id) - raise RuntimeError("Could not find reason for %s" % e.event_id) + # TODO: e is not in the current state, so we should + # construct some proof of that. + continue reason_map[e.event_id] = reason -- cgit 1.4.1 From cc0532a4bf328478d508242d830433a33c4f22b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 15:16:26 +0000 Subject: Explicitly list the RejectedReasons that we can prove --- synapse/handlers/federation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b13b7c7701..0f9c82fd06 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -860,9 +860,14 @@ class FederationHandler(BaseHandler): # Only do auth resolution if we have something new to say. # We can't rove an auth failure. do_resolution = False + + provable = [ + RejectedReason.NOT_ANCESTOR, RejectedReason.NOT_ANCESTOR, + ] + for e_id in different_auth: if e_id in have_events: - if have_events[e_id] != RejectedReason.AUTH_ERROR: + if have_events[e_id] in provable: do_resolution = True break -- cgit 1.4.1 From 55a186485c8def40e578201bb33457c9b7df9dde Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 6 Feb 2015 15:58:40 +0000 Subject: SYN-258: get_recent_events_for_room only accepts stream tokens, convert the topological token to a stream token before passing it to get_recent_events_for_room --- synapse/handlers/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 962686f4bb..439164ae39 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -298,15 +298,17 @@ class SyncHandler(BaseHandler): load_limit = max(sync_config.limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key + end_key = room_key while limited and len(recents) < sync_config.limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, - end_token=room_key, + end_token=end_key, ) (room_key, _) = keys + end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents -- cgit 1.4.1 From 20db147ef377cefb2b5cc138c2bd95291e26f3cc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 6 Feb 2015 15:58:40 +0000 Subject: SYN-258: get_recent_events_for_room only accepts stream tokens, convert the topological token to a stream token before passing it to get_recent_events_for_room --- synapse/handlers/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 962686f4bb..439164ae39 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -298,15 +298,17 @@ class SyncHandler(BaseHandler): load_limit = max(sync_config.limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key + end_key = room_key while limited and len(recents) < sync_config.limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, - end_token=room_key, + end_token=end_key, ) (room_key, _) = keys + end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents -- cgit 1.4.1 From 73a680b2a87b309bf05b4afe2685aaba70201305 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 17:10:04 +0000 Subject: Add errcodes for appservice registrations. --- synapse/handlers/register.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 60821edb05..10ab9bc99e 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -18,7 +18,8 @@ from twisted.internet import defer from synapse.types import UserID from synapse.api.errors import ( - SynapseError, RegistrationError, InvalidCaptchaError, CodeMessageException + AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError, + CodeMessageException ) from ._base import BaseHandler import synapse.util.stringutils as stringutils @@ -111,10 +112,11 @@ class RegistrationHandler(BaseHandler): user_id = user.to_string() service = yield self.store.get_app_service_by_token(as_token) if not service: - raise SynapseError(403, "Invalid application service token.") + raise AuthError(403, "Invalid application service token.") if not service.is_interested_in_user(user_id): raise SynapseError( - 400, "Invalid user localpart for this application service." + 400, "Invalid user localpart for this application service.", + errcode=Codes.EXCLUSIVE ) token = self._generate_token(user_id) yield self.store.register( @@ -181,7 +183,8 @@ class RegistrationHandler(BaseHandler): ] if len(interested_services) > 0: raise SynapseError( - 400, "This user ID is reserved by an application service." + 400, "This user ID is reserved by an application service.", + errcode=Codes.EXCLUSIVE ) def _generate_token(self, user_id): -- cgit 1.4.1 From 582019f870adbc4a8a8a9ef97b527e0fead77761 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 13:32:14 +0000 Subject: ...and here's the actual impl. git fail. --- synapse/handlers/register.py | 14 ++++++++ synapse/rest/media/v1/upload_resource.py | 57 ++++++++++++++++++-------------- 2 files changed, 46 insertions(+), 25 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 66a89c10b2..2b9d860084 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -99,6 +99,20 @@ class RegistrationHandler(BaseHandler): raise RegistrationError( 500, "Cannot generate user ID.") + # create a default avatar for the user + # XXX: ideally clients would explicitly specify one, but given they don't + # and we want consistent and pretty identicons for random users, we'll + # do it here. + auth_user = UserID.from_string(user_id) + identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) + upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) + content_uri = yield upload_resource.create_content( + "image/png", None, identicon_bytes, len(identicon_bytes), auth_user + ) + profile_handler = self.hs.get_handlers().profile_handler + profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + defer.returnValue((user_id, token)) @defer.inlineCallbacks diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index b939a30e19..5b42782331 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -38,6 +38,35 @@ class UploadResource(BaseMediaResource): def render_OPTIONS(self, request): respond_with_json(request, 200, {}, send_cors=True) return NOT_DONE_YET + + @defer.inlineCallbacks + def create_content(self, media_type, upload_name, content, content_length, auth_user): + media_id = random_string(24) + + fname = self.filepaths.local_media_filepath(media_id) + self._makedirs(fname) + + # This shouldn't block for very long because the content will have + # already been uploaded at this point. + with open(fname, "wb") as f: + f.write(content) + + yield self.store.store_local_media( + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=upload_name, + media_length=content_length, + user_id=auth_user, + ) + media_info = { + "media_type": media_type, + "media_length": content_length, + } + + yield self._generate_local_thumbnails(media_id, media_info) + + defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) @defer.inlineCallbacks def _async_render_POST(self, request): @@ -70,32 +99,10 @@ class UploadResource(BaseMediaResource): # disposition = headers.getRawHeaders("Content-Disposition")[0] # TODO(markjh): parse content-dispostion - media_id = random_string(24) - - fname = self.filepaths.local_media_filepath(media_id) - self._makedirs(fname) - - # This shouldn't block for very long because the content will have - # already been uploaded at this point. - with open(fname, "wb") as f: - f.write(request.content.read()) - - yield self.store.store_local_media( - media_id=media_id, - media_type=media_type, - time_now_ms=self.clock.time_msec(), - upload_name=None, - media_length=content_length, - user_id=auth_user, + content_uri = yield self.create_content( + media_type, None, request.content.read(), + content_length, auth_user ) - media_info = { - "media_type": media_type, - "media_length": content_length, - } - - yield self._generate_local_thumbnails(media_id, media_info) - - content_uri = "mxc://%s/%s" % (self.server_name, media_id) respond_with_json( request, 200, {"content_uri": content_uri}, send_cors=True -- cgit 1.4.1 From 37b6b880ef084f2cdd3cce1a275aa46764131c91 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 21:24:08 +0000 Subject: don't give up if we can't create default avatars during tests --- synapse/handlers/register.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 2b9d860084..4f06c487b1 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -103,15 +103,18 @@ class RegistrationHandler(BaseHandler): # XXX: ideally clients would explicitly specify one, but given they don't # and we want consistent and pretty identicons for random users, we'll # do it here. - auth_user = UserID.from_string(user_id) - identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) - upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) - identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) - content_uri = yield upload_resource.create_content( - "image/png", None, identicon_bytes, len(identicon_bytes), auth_user - ) - profile_handler = self.hs.get_handlers().profile_handler - profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + try: + auth_user = UserID.from_string(user_id) + identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) + upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) + content_uri = yield upload_resource.create_content( + "image/png", None, identicon_bytes, len(identicon_bytes), auth_user + ) + profile_handler = self.hs.get_handlers().profile_handler + profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + except NotImplementedError: + pass # make tests pass without messing around creating default avatars defer.returnValue((user_id, token)) -- cgit 1.4.1 From ac3183caaa66b750996d90c0ac9ed430f623909c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 12:03:37 +0000 Subject: Register a user account for the AS when the AS registers. Add 'sender' column to AS table. --- synapse/appservice/__init__.py | 3 ++- synapse/handlers/appservice.py | 8 +++++++- synapse/storage/appservice.py | 5 +++-- synapse/storage/schema/application_services.sql | 1 + synapse/storage/schema/delta/v14.sql | 1 + 5 files changed, 14 insertions(+), 4 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 46d46a5a48..fb9bfffe5d 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -35,10 +35,11 @@ class ApplicationService(object): NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None, hs_token=None, - txn_id=None): + sender=None, txn_id=None): self.token = token self.url = url self.hs_token = hs_token + self.sender = sender self.namespaces = self._check_namespaces(namespaces) self.txn_id = txn_id diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index fa810b9a98..5071a12eb1 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -52,8 +52,14 @@ class ApplicationServicesHandler(object): "Consult the home server admin.", errcode=Codes.FORBIDDEN ) - logger.info("Updating application service info...") + app_service.hs_token = self._generate_hs_token() + + # create a sender for this application service which is used when + # creating rooms, etc.. + account = yield self.hs.get_handlers().registration_handler.register() + app_service.sender = account[0] + yield self.store.update_app_service(app_service) defer.returnValue(app_service) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 3c8bf9ad0d..eef77e737e 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -130,8 +130,9 @@ class ApplicationServiceStore(SQLBaseStore): return False txn.execute( - "UPDATE application_services SET url=?, hs_token=? WHERE id=?", - (service.url, service.hs_token, as_id,) + "UPDATE application_services SET url=?, hs_token=?, sender=? " + "WHERE id=?", + (service.url, service.hs_token, service.sender, as_id,) ) # cleanup regex txn.execute( diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql index 03b5a10c8a..e491ad5aec 100644 --- a/synapse/storage/schema/application_services.sql +++ b/synapse/storage/schema/application_services.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS application_services( url TEXT, token TEXT, hs_token TEXT, + sender TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); diff --git a/synapse/storage/schema/delta/v14.sql b/synapse/storage/schema/delta/v14.sql index 03b5a10c8a..e491ad5aec 100644 --- a/synapse/storage/schema/delta/v14.sql +++ b/synapse/storage/schema/delta/v14.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS application_services( url TEXT, token TEXT, hs_token TEXT, + sender TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); -- cgit 1.4.1 From f7cac2f7b699fb3bb58c02fa1828aeec6c319f01 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 15:01:28 +0000 Subject: Fix bugs so lazy room joining works as intended. --- synapse/appservice/api.py | 6 +++--- synapse/handlers/appservice.py | 6 +++--- synapse/handlers/directory.py | 23 +++++++++++++++++------ 3 files changed, 23 insertions(+), 12 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 9cce4e0973..15ac1e27fc 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -41,7 +41,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": service.hs_token }) - if response: # just an empty json object + if response is not None: # just an empty json object defer.returnValue(True) except CodeMessageException as e: if e.code == 404: @@ -60,13 +60,13 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": service.hs_token }) - if response: # just an empty json object + if response is not None: # just an empty json object defer.returnValue(True) except CodeMessageException as e: + logger.warning("query_alias to %s received %s", uri, e.code) if e.code == 404: defer.returnValue(False) return - logger.warning("query_alias to %s received %s", uri, e.code) except Exception as ex: logger.warning("query_alias to %s threw exception %s", uri, ex) defer.returnValue(False) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 5071a12eb1..8591a77bf3 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -124,15 +124,15 @@ class ApplicationServicesHandler(object): namedtuple: with keys "room_id" and "servers" or None if no association can be found. """ - room_alias = room_alias.to_string() + room_alias_str = room_alias.to_string() alias_query_services = yield self._get_services_for_event( event=None, restrict_to=ApplicationService.NS_ALIASES, - alias_list=[room_alias] + alias_list=[room_alias_str] ) for alias_service in alias_query_services: is_known_alias = yield self.appservice_api.query_alias( - alias_service, room_alias + alias_service, room_alias_str ) if is_known_alias: # the alias exists now so don't query more ASes. diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 22c6391a15..87bc12c983 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -64,8 +64,11 @@ class DirectoryHandler(BaseHandler): # association creation for human users # TODO(erikj): Do user auth. - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: + can_create = yield self.can_modify_alias( + room_alias, + user_id=user_id + ) + if not can_create: raise SynapseError( 400, "This alias is reserved by an application service.", errcode=Codes.EXCLUSIVE @@ -91,8 +94,11 @@ class DirectoryHandler(BaseHandler): # TODO Check if server admin - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: + can_delete = yield self.can_modify_alias( + room_alias, + user_id=user_id + ) + if not can_delete: raise SynapseError( 400, "This alias is reserved by an application service.", errcode=Codes.EXCLUSIVE @@ -228,9 +234,14 @@ class DirectoryHandler(BaseHandler): defer.returnValue(result) @defer.inlineCallbacks - def is_alias_exclusive_to_appservices(self, alias): + def can_modify_alias(self, alias, user_id=None): services = yield self.store.get_app_services() interested_services = [ s for s in services if s.is_interested_in_alias(alias.to_string()) ] - defer.returnValue(len(interested_services) > 0) + for service in interested_services: + if user_id == service.sender: + # this user IS the app service + defer.returnValue(True) + return + defer.returnValue(len(interested_services) == 0) -- cgit 1.4.1 From d94f682a4c3e7bab5079d516582f0ee44a3d3f06 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 17:41:29 +0000 Subject: During room intial sync, only calculate current state once. --- synapse/api/auth.py | 21 ++++++++++++++------- synapse/handlers/message.py | 32 ++++++++++++++++++++++---------- synapse/handlers/sync.py | 9 ++++++--- synapse/state.py | 2 +- 4 files changed, 43 insertions(+), 21 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 7105ee21dc..0054745363 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -89,12 +89,19 @@ class Auth(object): raise @defer.inlineCallbacks - def check_joined_room(self, room_id, user_id): - member = yield self.state.get_current_state( - room_id=room_id, - event_type=EventTypes.Member, - state_key=user_id - ) + def check_joined_room(self, room_id, user_id, current_state=None): + if current_state: + member = current_state.get( + (EventTypes.Member, user_id), + None + ) + else: + member = yield self.state.get_current_state( + room_id=room_id, + event_type=EventTypes.Member, + state_key=user_id + ) + self._check_joined_room(member, user_id, room_id) defer.returnValue(member) @@ -102,7 +109,7 @@ class Auth(object): def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) - for event in curr_state: + for event in curr_state.values(): if event.type == EventTypes.Member: try: if UserID.from_string(event.state_key).domain != host: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3f51f38f18..3355adefcf 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -35,6 +35,7 @@ class MessageHandler(BaseHandler): def __init__(self, hs): super(MessageHandler, self).__init__(hs) self.hs = hs + self.state = hs.get_state_handler() self.clock = hs.get_clock() self.validator = EventValidator() @@ -225,7 +226,9 @@ class MessageHandler(BaseHandler): # TODO: This is duplicating logic from snapshot_all_rooms current_state = yield self.state_handler.get_current_state(room_id) now = self.clock.time_msec() - defer.returnValue([serialize_event(c, now) for c in current_state]) + defer.returnValue( + [serialize_event(c, now) for c in current_state.values()] + ) @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, @@ -313,7 +316,7 @@ class MessageHandler(BaseHandler): ) d["state"] = [ serialize_event(c, time_now, as_client_event) - for c in current_state + for c in current_state.values() ] except: logger.exception("Failed to get snapshot") @@ -329,7 +332,14 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def room_initial_sync(self, user_id, room_id, pagin_config=None, feedback=False): - yield self.auth.check_joined_room(room_id, user_id) + current_state = yield self.state.get_current_state( + room_id=room_id, + ) + + yield self.auth.check_joined_room( + room_id, user_id, + current_state=current_state + ) # TODO(paul): I wish I was called with user objects not user_id # strings... @@ -337,13 +347,12 @@ class MessageHandler(BaseHandler): # TODO: These concurrently time_now = self.clock.time_msec() - state_tuples = yield self.state_handler.get_current_state(room_id) - state = [serialize_event(x, time_now) for x in state_tuples] + state = [ + serialize_event(x, time_now) + for x in current_state.values() + ] - member_event = (yield self.store.get_room_member( - user_id=user_id, - room_id=room_id - )) + member_event = current_state.get((EventTypes.Member, user_id,)) now_token = yield self.hs.get_event_sources().get_current_token() @@ -360,7 +369,10 @@ class MessageHandler(BaseHandler): start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) - room_members = yield self.store.get_room_members(room_id) + room_members = [ + m for m in current_state.values() + if m.type == EventTypes.Member + ] presence_handler = self.hs.get_handlers().presence_handler presence = [] diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 439164ae39..5af90cc5d1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -175,9 +175,10 @@ class SyncHandler(BaseHandler): room_id, sync_config, now_token, ) - current_state_events = yield self.state_handler.get_current_state( + current_state = yield self.state_handler.get_current_state( room_id ) + current_state_events = current_state.values() defer.returnValue(RoomSyncResult( room_id=room_id, @@ -347,9 +348,10 @@ class SyncHandler(BaseHandler): # TODO(mjark): This seems racy since this isn't being passed a # token to indicate what point in the stream this is - current_state_events = yield self.state_handler.get_current_state( + current_state = yield self.state_handler.get_current_state( room_id ) + current_state_events = current_state.values() state_at_previous_sync = yield self.get_state_at_previous_sync( room_id, since_token=since_token @@ -431,6 +433,7 @@ class SyncHandler(BaseHandler): joined = True if joined: - state_delta = yield self.state_handler.get_current_state(room_id) + res = yield self.state_handler.get_current_state(room_id) + state_delta = res.values() defer.returnValue(state_delta) diff --git a/synapse/state.py b/synapse/state.py index 695a5e7ac4..54380b9e5c 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -76,7 +76,7 @@ class StateHandler(object): defer.returnValue(res[1].get((event_type, state_key))) return - defer.returnValue(res[1].values()) + defer.returnValue(res[1]) @defer.inlineCallbacks def compute_event_context(self, event, old_state=None): -- cgit 1.4.1 From b085fac7353e1cd395b89f9334c8273a8e996f48 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 16:30:48 +0000 Subject: Code-style fixes --- synapse/handlers/presence.py | 4 +++- synapse/handlers/register.py | 13 ++++++++----- synapse/push/__init__.py | 8 +++++--- synapse/push/baserules.py | 11 ++++++----- synapse/push/rulekinds.py | 12 ++++++------ synapse/python_dependencies.py | 6 ++++-- synapse/rest/client/v1/push_rule.py | 18 +++++++++++++----- synapse/rest/client/v1/pusher.py | 4 ++-- synapse/rest/media/v1/upload_resource.py | 7 ++++--- synapse/storage/_base.py | 6 +++--- synapse/storage/push_rule.py | 4 +++- 11 files changed, 57 insertions(+), 36 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index cd0798c2b0..6a266ee0fe 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -658,7 +658,9 @@ class PresenceHandler(BaseHandler): observers = set(self._remote_recvmap.get(user, set())) if observers: - logger.debug(" | %d interested local observers %r", len(observers), observers) + logger.debug( + " | %d interested local observers %r", len(observers), observers + ) rm_handler = self.homeserver.get_handlers().room_member_handler room_ids = yield rm_handler.get_rooms_for_user(user) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 4f06c487b1..0247327eb9 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -105,17 +105,20 @@ class RegistrationHandler(BaseHandler): # do it here. try: auth_user = UserID.from_string(user_id) - identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) - upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + media_repository = self.hs.get_resource_for_media_repository() + identicon_resource = media_repository.getChildWithDefault("identicon", None) + upload_resource = media_repository.getChildWithDefault("upload", None) identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) content_uri = yield upload_resource.create_content( "image/png", None, identicon_bytes, len(identicon_bytes), auth_user ) profile_handler = self.hs.get_handlers().profile_handler - profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + profile_handler.set_avatar_url( + auth_user, auth_user, ("%s#auto" % (content_uri,)) + ) except NotImplementedError: - pass # make tests pass without messing around creating default avatars - + pass # make tests pass without messing around creating default avatars + defer.returnValue((user_id, token)) @defer.inlineCallbacks diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 6f143a5df9..418a348a58 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -140,7 +140,7 @@ class Pusher(object): lambda x: ('[%s%s]' % (x.group(1) and '^' or '', re.sub(r'\\\-', '-', x.group(2)))), r) return r - + def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): if condition['kind'] == 'event_match': if 'pattern' not in condition: @@ -170,8 +170,10 @@ class Pusher(object): return False if not display_name: return False - return re.search("\b%s\b" % re.escape(display_name), - ev['content']['body'], flags=re.IGNORECASE) is not None + return re.search( + "\b%s\b" % re.escape(display_name), ev['content']['body'], + flags=re.IGNORECASE + ) is not None elif condition['kind'] == 'room_member_count': if 'is' not in condition: diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 37878f1e0b..162d265f66 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -1,5 +1,6 @@ from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP + def list_with_base_rules(rawrules, user_name): ruleslist = [] @@ -9,9 +10,9 @@ def list_with_base_rules(rawrules, user_name): if r['priority_class'] < current_prio_class: while r['priority_class'] < current_prio_class: ruleslist.extend(make_base_rules( - user_name, - PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) - ) + user_name, + PRIORITY_CLASS_INVERSE_MAP[current_prio_class] + )) current_prio_class -= 1 ruleslist.append(r) @@ -19,8 +20,8 @@ def list_with_base_rules(rawrules, user_name): while current_prio_class > 0: ruleslist.extend(make_base_rules( user_name, - PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) - ) + PRIORITY_CLASS_INVERSE_MAP[current_prio_class] + )) current_prio_class -= 1 return ruleslist diff --git a/synapse/push/rulekinds.py b/synapse/push/rulekinds.py index 763bdee58e..660aa4e10e 100644 --- a/synapse/push/rulekinds.py +++ b/synapse/push/rulekinds.py @@ -1,8 +1,8 @@ PRIORITY_CLASS_MAP = { - 'underride': 1, - 'sender': 2, - 'room': 3, - 'content': 4, - 'override': 5, - } + 'underride': 1, + 'sender': 2, + 'room': 3, + 'content': 4, + 'override': 5, +} PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a89d618606..fd68da9dfb 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -19,10 +19,11 @@ REQUIREMENTS = { "pydenticon": ["pydenticon"], } + def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) -DEPENDENCY_LINKS=[ +DEPENDENCY_LINKS = [ github_link( project="matrix-org/syutil", version="v0.0.2", @@ -101,6 +102,7 @@ def check_requirements(): % (dependency, file_path, version, required_version) ) + def list_requirements(): result = [] linked = [] @@ -111,7 +113,7 @@ def list_requirements(): for requirement in REQUIREMENTS: is_linked = False for link in linked: - if requirement.replace('-','_').startswith(link): + if requirement.replace('-', '_').startswith(link): is_linked = True if not is_linked: result.append(requirement) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index d43ade39dd..c4e7dfcf0e 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -15,12 +15,17 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError, \ - StoreError +from synapse.api.errors import ( + SynapseError, Codes, UnrecognizedRequestError, NotFoundError, StoreError +) from .base import ClientV1RestServlet, client_path_pattern -from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException +from synapse.storage.push_rule import ( + InconsistentRuleException, RuleNotFoundException +) import synapse.push.baserules as baserules -from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP +from synapse.push.rulekinds import ( + PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP +) import json @@ -105,7 +110,9 @@ class PushRuleRestServlet(ClientV1RestServlet): # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference - rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name( + user.to_string() + ) for r in rawrules: r["conditions"] = json.loads(r["conditions"]) @@ -383,6 +390,7 @@ def _namespaced_rule_id_from_spec(spec): def _rule_id_from_namespaced(in_rule_id): return in_rule_id.split('/')[-1] + class InvalidRuleException(Exception): pass diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index e10d2576d2..80e9939b79 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -34,8 +34,8 @@ class PusherRestServlet(ClientV1RestServlet): pusher_pool = self.hs.get_pusherpool() if ('pushkey' in content and 'app_id' in content - and 'kind' in content and - content['kind'] is None): + and 'kind' in content and + content['kind'] is None): yield pusher_pool.remove_pusher( content['app_id'], content['pushkey'] ) diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 5b42782331..6df52ca434 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -38,9 +38,10 @@ class UploadResource(BaseMediaResource): def render_OPTIONS(self, request): respond_with_json(request, 200, {}, send_cors=True) return NOT_DONE_YET - + @defer.inlineCallbacks - def create_content(self, media_type, upload_name, content, content_length, auth_user): + def create_content(self, media_type, upload_name, content, content_length, + auth_user): media_id = random_string(24) fname = self.filepaths.local_media_filepath(media_id) @@ -65,7 +66,7 @@ class UploadResource(BaseMediaResource): } yield self._generate_local_thumbnails(media_id, media_info) - + defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) @defer.inlineCallbacks diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 36455ef93c..3e1ab0a159 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -84,7 +84,7 @@ class PerformanceCounters(object): def update(self, key, start_time, end_time=None): if end_time is None: - end_time = time.time() * 1000; + end_time = time.time() * 1000 duration = end_time - start_time count, cum_time = self.current_counters.get(key, (0, 0)) count += 1 @@ -588,7 +588,7 @@ class SQLBaseStore(object): "LIMIT 1 " ) - start_time = time.time() * 1000; + start_time = time.time() * 1000 txn.execute(sql, (event_id,)) @@ -613,7 +613,7 @@ class SQLBaseStore(object): def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False): - start_time = time.time() * 1000; + start_time = time.time() * 1000 update_counter = self._get_event_counters.update d = json.loads(js) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 30e23445d9..620de71398 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -91,7 +91,9 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, relative_to_rule)) res = txn.fetchall() if not res: - raise RuleNotFoundException("before/after rule not found: %s" % (relative_to_rule)) + raise RuleNotFoundException( + "before/after rule not found: %s" % (relative_to_rule,) + ) priority_class, base_rule_priority = res[0] if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: -- cgit 1.4.1 From 84a769cdb7321481954c386a040d7e7ff504d02b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 17:58:36 +0000 Subject: Fix code-style --- synapse/app/homeserver.py | 2 +- synapse/crypto/keyclient.py | 4 ++-- synapse/handlers/presence.py | 8 ++++---- synapse/handlers/sync.py | 2 +- synapse/rest/client/v1/directory.py | 2 +- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/media/v1/upload_resource.py | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8976ff2e82..ff6680e7da 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -134,7 +134,7 @@ class SynapseHomeServer(HomeServer): logger.info("Attaching %s to path %s", resource, full_path) last_resource = self.root_resource for path_seg in full_path.split('/')[1:-1]: - if not path_seg in last_resource.listNames(): + if path_seg not in last_resource.listNames(): # resource doesn't exist, so make a "dummy resource" child_resource = Resource() last_resource.putChild(path_seg, child_resource) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index cdb6279764..cd12349f67 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -75,7 +75,7 @@ class SynapseKeyClientProtocol(HTTPClient): def handleStatus(self, version, status, message): if status != b"200": - #logger.info("Non-200 response from %s: %s %s", + # logger.info("Non-200 response from %s: %s %s", # self.transport.getHost(), status, message) self.transport.abortConnection() @@ -83,7 +83,7 @@ class SynapseKeyClientProtocol(HTTPClient): try: json_response = json.loads(response_body_bytes) except ValueError: - #logger.info("Invalid JSON response from %s", + # logger.info("Invalid JSON response from %s", # self.transport.getHost()) self.transport.abortConnection() return diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6a266ee0fe..59287010ed 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -457,9 +457,9 @@ class PresenceHandler(BaseHandler): if state is None: state = yield self.store.get_presence_state(user.localpart) else: -# statuscache = self._get_or_make_usercache(user) -# self._user_cachemap_latest_serial += 1 -# statuscache.update(state, self._user_cachemap_latest_serial) + # statuscache = self._get_or_make_usercache(user) + # self._user_cachemap_latest_serial += 1 + # statuscache.update(state, self._user_cachemap_latest_serial) pass yield self.push_update_to_local_and_remote( @@ -709,7 +709,7 @@ class PresenceHandler(BaseHandler): # TODO(paul) permissions checks - if not user in self._remote_sendmap: + if user not in self._remote_sendmap: self._remote_sendmap[user] = set() self._remote_sendmap[user].add(origin) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5af90cc5d1..7883bbd834 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -114,7 +114,7 @@ class SyncHandler(BaseHandler): if sync_config.gap: return self.incremental_sync_with_gap(sync_config, since_token) else: - #TODO(mjark): Handle gapless sync + # TODO(mjark): Handle gapless sync raise NotImplementedError() @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 8f65efec5f..8ed7e2d669 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -48,7 +48,7 @@ class ClientDirectoryServer(ClientV1RestServlet): user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) - if not "room_id" in content: + if "room_id" not in content: raise SynapseError(400, "Missing room_id key", errcode=Codes.BAD_JSON) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 81d5cf8ead..3056ec45cf 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -118,7 +118,7 @@ class SyncRestServlet(RestServlet): except: filter = Filter({}) # filter = filter.apply_overrides(http_request) - #if filter.matches(event): + # if filter.matches(event): # # stuff sync_config = SyncConfig( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 6df52ca434..e5aba3af4c 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -96,8 +96,8 @@ class UploadResource(BaseMediaResource): code=400, ) - #if headers.hasHeader("Content-Disposition"): - # disposition = headers.getRawHeaders("Content-Disposition")[0] + # if headers.hasHeader("Content-Disposition"): + # disposition = headers.getRawHeaders("Content-Disposition")[0] # TODO(markjh): parse content-dispostion content_uri = yield self.create_content( -- cgit 1.4.1 From c7783d6feec9b69c24f3303cbb51cce3e6b8ffb3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:36:08 +0000 Subject: Notify ASes for events sent by other users in a room which an AS user is a part of. --- synapse/appservice/__init__.py | 17 +++++++++++++---- synapse/handlers/appservice.py | 21 ++++++++++++++++----- synapse/rest/appservice/v1/__init__.py | 4 ++-- tests/appservice/test_appservice.py | 25 +++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 11 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index fb9bfffe5d..381b4cfc4a 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -74,7 +74,7 @@ class ApplicationService(object): return True return False - def _matches_user(self, event): + def _matches_user(self, event, member_list): if (hasattr(event, "sender") and self.is_interested_in_user(event.sender)): return True @@ -83,6 +83,10 @@ class ApplicationService(object): and hasattr(event, "state_key") and self.is_interested_in_user(event.state_key)): return True + # check joined member events + for member in member_list: + if self.is_interested_in_user(member.state_key): + return True return False def _matches_room_id(self, event): @@ -96,7 +100,8 @@ class ApplicationService(object): return True return False - def is_interested(self, event, restrict_to=None, aliases_for_event=None): + def is_interested(self, event, restrict_to=None, aliases_for_event=None, + member_list=None): """Check if this service is interested in this event. Args: @@ -104,18 +109,22 @@ class ApplicationService(object): restrict_to(str): The namespace to restrict regex tests to. aliases_for_event(list): A list of all the known room aliases for this event. + member_list(list): A list of all joined room members in this room. Returns: bool: True if this service would like to know about this event. """ if aliases_for_event is None: aliases_for_event = [] + if member_list is None: + member_list = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: # this is a programming error, so fail early and raise a general # exception raise Exception("Unexpected restrict_to value: %s". restrict_to) if not restrict_to: - return (self._matches_user(event) + return (self._matches_user(event, member_list) or self._matches_aliases(event, aliases_for_event) or self._matches_room_id(event)) elif restrict_to == ApplicationService.NS_ALIASES: @@ -123,7 +132,7 @@ class ApplicationService(object): elif restrict_to == ApplicationService.NS_ROOMS: return self._matches_room_id(event) elif restrict_to == ApplicationService.NS_USERS: - return self._matches_user(event) + return self._matches_user(event, member_list) def is_interested_in_user(self, user_id): return self._matches_regex(user_id, ApplicationService.NS_USERS) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8591a77bf3..2c488a46f6 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, Membership from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.types import UserID @@ -154,14 +154,25 @@ class ApplicationServicesHandler(object): list: A list of services interested in this event based on the service regex. """ - # We need to know the aliases associated with this event.room_id, if any - if not alias_list: - alias_list = yield self.store.get_aliases_for_room(event.room_id) + member_list = None + if hasattr(event, "room_id"): + # We need to know the aliases associated with this event.room_id, + # if any. + if not alias_list: + alias_list = yield self.store.get_aliases_for_room( + event.room_id + ) + # We need to know the members associated with this event.room_id, + # if any. + member_list = yield self.store.get_room_members( + room_id=event.room_id, + membership=Membership.JOIN + ) services = yield self.store.get_app_services() interested_list = [ s for s in services if ( - s.is_interested(event, restrict_to, alias_list) + s.is_interested(event, restrict_to, alias_list, member_list) ) ] defer.returnValue(interested_list) diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py index bf243b6180..a7877609ad 100644 --- a/synapse/rest/appservice/v1/__init__.py +++ b/synapse/rest/appservice/v1/__init__.py @@ -21,9 +21,9 @@ class AppServiceRestResource(JsonResource): """A resource for version 1 of the matrix application service API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod def register_servlets(appservice_resource, hs): - register.register_servlets(hs, appservice_resource) \ No newline at end of file + register.register_servlets(hs, appservice_resource) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index c0aaf12785..d12e4f2644 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -143,3 +143,28 @@ class ApplicationServiceTestCase(unittest.TestCase): restrict_to=ApplicationService.NS_USERS, aliases_for_event=["#xmpp_barfoo:matrix.org"] )) + + def test_member_list_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + join_list = [ + Mock( + type="m.room.member", room_id="!foo:bar", sender="@alice:here", + state_key="@alice:here" + ), + Mock( + type="m.room.member", room_id="!foo:bar", sender="@irc_fo:here", + state_key="@irc_fo:here" # AS user + ), + Mock( + type="m.room.member", room_id="!foo:bar", sender="@bob:here", + state_key="@bob:here" + ) + ] + + self.event.sender = "@xmpp_foobar:matrix.org" + self.assertTrue(self.service.is_interested( + event=self.event, + member_list=join_list + )) -- cgit 1.4.1 From fd40d992adfb8b63f6e925dad030c63498501408 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:41:33 +0000 Subject: PEP8-ify --- synapse/appservice/api.py | 2 -- synapse/handlers/directory.py | 3 --- synapse/rest/appservice/v1/register.py | 2 +- synapse/storage/appservice.py | 3 --- 4 files changed, 1 insertion(+), 9 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 15ac1e27fc..6192813c03 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -71,7 +71,6 @@ class ApplicationServiceApi(SimpleHttpClient): logger.warning("query_alias to %s threw exception %s", uri, ex) defer.returnValue(False) - @defer.inlineCallbacks def push_bulk(self, service, events): events = self._serialize(events) @@ -107,4 +106,3 @@ class ApplicationServiceApi(SimpleHttpClient): return [ serialize_event(e, time_now, as_client_event=True) for e in events ] - diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 87bc12c983..20ab9e269c 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -58,7 +58,6 @@ class DirectoryHandler(BaseHandler): servers ) - @defer.inlineCallbacks def create_association(self, user_id, room_alias, room_id, servers=None): # association creation for human users @@ -75,7 +74,6 @@ class DirectoryHandler(BaseHandler): ) yield self._create_association(room_alias, room_id, servers) - @defer.inlineCallbacks def create_appservice_association(self, service, room_alias, room_id, servers=None): @@ -127,7 +125,6 @@ class DirectoryHandler(BaseHandler): # if room_id: # yield self._update_room_alias_events(user_id, room_id) - @defer.inlineCallbacks def get_association(self, room_alias): room_id = None diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index d3d5aef220..3bd0c1220c 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -65,7 +65,7 @@ class RegisterRestServlet(AppServiceRestServlet): hs_token = app_service.hs_token defer.returnValue((200, { - "hs_token": hs_token + "hs_token": hs_token })) def _parse_namespace(self, target_ns, origin_ns, ns): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index ba31c68595..d941b1f387 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -97,7 +97,6 @@ class ApplicationServiceStore(SQLBaseStore): # allocate new ASes. It relies on the server admin inserting the AS # token into the database manually. - if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") @@ -186,7 +185,6 @@ class ApplicationServiceStore(SQLBaseStore): # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. - @defer.inlineCallbacks def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" @@ -244,4 +242,3 @@ class ApplicationServiceStore(SQLBaseStore): hs_token=service["hs_token"], sender=service["sender"] )) - -- cgit 1.4.1 From ddb816cf60ca1b0c0f9bfab5df233a010ac309a3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 15:44:28 +0000 Subject: Don't unfreeze when using FreezeEvent.get_dict, as we are using a JSONEncoder that understands FrozenDict --- synapse/events/__init__.py | 4 ---- synapse/federation/persistence.py | 7 ++++--- synapse/handlers/federation.py | 15 ++++++++++++++- synapse/handlers/room.py | 4 +++- synapse/storage/__init__.py | 11 +++++++++-- synapse/util/frozenutils.py | 8 ++++++-- 6 files changed, 36 insertions(+), 13 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 8f0c6e959f..f4ec8cd18c 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -140,10 +140,6 @@ class FrozenEvent(EventBase): return e - def get_dict(self): - # We need to unfreeze what we return - return unfreeze(super(FrozenEvent, self).get_dict()) - def __str__(self): return self.__repr__() diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 8a1afc0ca5..76a9dcd777 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -23,7 +23,8 @@ from twisted.internet import defer from synapse.util.logutils import log_function -import simplejson as json +from syutil.jsonutil import encode_canonical_json + import logging @@ -70,7 +71,7 @@ class TransactionActions(object): transaction.transaction_id, transaction.origin, code, - json.dumps(response) + encode_canonical_json(response) ) @defer.inlineCallbacks @@ -100,5 +101,5 @@ class TransactionActions(object): transaction.transaction_id, transaction.destination, response_code, - json.dumps(response_dict) + encode_canonical_json(response_dict) ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0f9c82fd06..e36f0945ef 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -23,6 +23,7 @@ from synapse.api.errors import ( from synapse.api.constants import EventTypes, Membership, RejectedReason from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor +from synapse.util.frozenutils import unfreeze from synapse.crypto.event_signing import ( compute_event_signature, add_hashes_and_signatures, ) @@ -311,9 +312,12 @@ class FederationHandler(BaseHandler): self.room_queues[room_id] = [] builder = self.event_builder_factory.new( - event.get_pdu_json() + unfreeze(event.get_pdu_json()) ) + logger.info("Builder: %s", builder.get_pdu_json()) + logger.info("Content: %s", content) + handled_events = set() try: @@ -324,14 +328,21 @@ class FederationHandler(BaseHandler): if not hasattr(event, "signatures"): builder.signatures = {} + logger.info("Content befhahs: %s", builder.content) + add_hashes_and_signatures( builder, self.hs.hostname, self.hs.config.signing_key[0], ) + logger.info("Content aftet hah: %s", builder.content) + logger.info("Content pdu json: %s", builder.get_pdu_json()) + new_event = builder.build() + logger.info("Content after build: %s", new_event.content) + # Try the host we successfully got a response to /make_join/ # request first. try: @@ -340,6 +351,7 @@ class FederationHandler(BaseHandler): except ValueError: pass + logger.info(new_event.content) ret = yield self.replication_layer.send_join( target_hosts, new_event @@ -485,6 +497,7 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False + logger.info(event.content) context = yield self._handle_new_event(origin, event) logger.debug( diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0369b907a5..c685991a9f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -403,7 +403,9 @@ class RoomMemberHandler(BaseHandler): "membership": Membership.JOIN, "content": content, }) + logger.info("Before build: %s", builder.get_pdu_json()) event, context = yield self._create_new_client_event(builder) + logger.info("After build: %s", event.get_dict()) yield self._do_join(event, context, room_hosts=hosts, do_auth=True) @@ -462,7 +464,7 @@ class RoomMemberHandler(BaseHandler): room_hosts, room_id, event.user_id, - event.get_dict()["content"], # FIXME To get a non-frozen dict + event.content, # FIXME To get a non-frozen dict context ) else: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 924ea89035..94603ef826 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -298,12 +298,16 @@ class DataStore(RoomMemberStore, RoomStore, or_replace=True, ) + content = encode_canonical_json( + event.content + ).decode("UTF-8") + vals = { "topological_ordering": event.depth, "event_id": event.event_id, "type": event.type, "room_id": event.room_id, - "content": json.dumps(event.get_dict()["content"]), + "content": content, "processed": True, "outlier": outlier, "depth": event.depth, @@ -323,7 +327,10 @@ class DataStore(RoomMemberStore, RoomStore, "prev_events", ] } - vals["unrecognized_keys"] = json.dumps(unrec) + + vals["unrecognized_keys"] = encode_canonical_json( + unrec + ).decode("UTF-8") try: self._simple_insert_txn( diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index a13a2015e4..9e10d37aec 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -21,6 +21,9 @@ def freeze(o): if t is dict: return frozendict({k: freeze(v) for k, v in o.items()}) + if t is frozendict: + return o + if t is str or t is unicode: return o @@ -33,10 +36,11 @@ def freeze(o): def unfreeze(o): - if isinstance(o, frozendict) or isinstance(o, dict): + t = type(o) + if t is dict or t is frozendict: return dict({k: unfreeze(v) for k, v in o.items()}) - if isinstance(o, basestring): + if t is str or t is unicode: return o try: -- cgit 1.4.1 From ed877d658500b736e7f7192fbeb64aac444ef4d8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 16:50:46 +0000 Subject: Remove debug logging --- synapse/handlers/federation.py | 10 ---------- synapse/handlers/room.py | 2 -- 2 files changed, 12 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e36f0945ef..0c35317bb8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -315,9 +315,6 @@ class FederationHandler(BaseHandler): unfreeze(event.get_pdu_json()) ) - logger.info("Builder: %s", builder.get_pdu_json()) - logger.info("Content: %s", content) - handled_events = set() try: @@ -328,21 +325,14 @@ class FederationHandler(BaseHandler): if not hasattr(event, "signatures"): builder.signatures = {} - logger.info("Content befhahs: %s", builder.content) - add_hashes_and_signatures( builder, self.hs.hostname, self.hs.config.signing_key[0], ) - logger.info("Content aftet hah: %s", builder.content) - logger.info("Content pdu json: %s", builder.get_pdu_json()) - new_event = builder.build() - logger.info("Content after build: %s", new_event.content) - # Try the host we successfully got a response to /make_join/ # request first. try: diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c685991a9f..914742d913 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -403,9 +403,7 @@ class RoomMemberHandler(BaseHandler): "membership": Membership.JOIN, "content": content, }) - logger.info("Before build: %s", builder.get_pdu_json()) event, context = yield self._create_new_client_event(builder) - logger.info("After build: %s", event.get_dict()) yield self._do_join(event, context, room_hosts=hosts, do_auth=True) -- cgit 1.4.1 From 76935078d1983aec9966f2eb245e4186dcee0386 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 16:51:22 +0000 Subject: Remove more debug logging --- synapse/handlers/federation.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0c35317bb8..77c81fe2da 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -341,7 +341,6 @@ class FederationHandler(BaseHandler): except ValueError: pass - logger.info(new_event.content) ret = yield self.replication_layer.send_join( target_hosts, new_event @@ -487,7 +486,6 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False - logger.info(event.content) context = yield self._handle_new_event(origin, event) logger.debug( -- cgit 1.4.1 From 860b1b4841618ec401c3652dcf19ac8b5d167f50 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:11:52 +0000 Subject: Only attempt to fetch presence state of JOINed members in room initialSync (SYN-202) --- synapse/handlers/message.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse/handlers') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3355adefcf..c6d6188d9d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -372,6 +372,7 @@ class MessageHandler(BaseHandler): room_members = [ m for m in current_state.values() if m.type == EventTypes.Member + and m.content["membership"] == Membership.JOIN ] presence_handler = self.hs.get_handlers().presence_handler -- cgit 1.4.1 From 29805213d1c20c89c20a4b3075c30597b4402338 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:12:57 +0000 Subject: Can now remove the FIXME too --- synapse/handlers/message.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c6d6188d9d..5f4cc1a84b 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -386,16 +386,9 @@ class MessageHandler(BaseHandler): ) presence.append(member_presence) except SynapseError as e: - if e.code == 404: - # FIXME: We are doing this as a warn since this gets hit a - # lot and spams the logs. Why is this happening? - logger.warn( - "Failed to get member presence of %r", m.user_id - ) - else: - logger.exception( - "Failed to get member presence of %r", m.user_id - ) + logger.exception( + "Failed to get member presence of %r", m.user_id + ) time_now = self.clock.time_msec() -- cgit 1.4.1 From 5f6e6530d0f43c637e527cfe1797801879dbc389 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:15:23 +0000 Subject: Appease pyflakes --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5f4cc1a84b..e028cfc2e9 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -385,7 +385,7 @@ class MessageHandler(BaseHandler): as_event=True, ) presence.append(member_presence) - except SynapseError as e: + except SynapseError: logger.exception( "Failed to get member presence of %r", m.user_id ) -- cgit 1.4.1 From 93d07c87dc9b837492fa0f691a77e716a2fe512b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:19:37 +0000 Subject: Reindent code to be less human-readable to keep pep8 from complaining --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index e028cfc2e9..7b9685be7f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -372,7 +372,7 @@ class MessageHandler(BaseHandler): room_members = [ m for m in current_state.values() if m.type == EventTypes.Member - and m.content["membership"] == Membership.JOIN + and m.content["membership"] == Membership.JOIN ] presence_handler = self.hs.get_handlers().presence_handler -- cgit 1.4.1 From 8c652a2b5f4ed07abd682543ad6a365c6d50b8d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 14:20:05 +0000 Subject: When we see a difference in current state, actually use state conflict resolution algorithm --- synapse/handlers/federation.py | 34 +++++++++++++++++++++++++++++++++ synapse/state.py | 43 ++++++++++++++++++++++++++++-------------- 2 files changed, 63 insertions(+), 14 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 77c81fe2da..5b225ffb57 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -858,6 +858,40 @@ class FederationHandler(BaseHandler): # Do auth conflict res. logger.debug("Different auth: %s", different_auth) + different_events = yield defer.gatherResults( + [ + self.store.get_event( + d, + allow_none=True, + allow_rejected=False, + ) + for d in different_auth + if d in have_events and not have_events[d] + ], + consumeErrors=True + ) + + if different_events: + local_view = dict(auth_events) + remote_view = dict(auth_events) + remote_view.update({ + (d.type, d.state_key) for d in different_events + }) + + new_state, _ = self.state.resolve_events( + [local_view, remote_view], + event + ) + + auth_events.update(new_state) + + current_state = set(e.event_id for e in auth_events.values()) + different_auth = event_auth_events - current_state + + context.current_state.update(auth_events) + context.state_group = None + + if different_auth and not event.internal_metadata.is_outlier(): # Only do auth resolution if we have something new to say. # We can't rove an auth failure. do_resolution = False diff --git a/synapse/state.py b/synapse/state.py index 98aaa2be53..fe5f3dc84b 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -259,13 +259,37 @@ class StateHandler(object): defer.returnValue((name, state, prev_states)) + new_state, prev_states = self._resolve_events( + state_groups.values(), event_type, state_key + ) + + if self._state_cache is not None: + cache = _StateCacheEntry( + state=new_state, + state_group=None, + ts=self.clock.time_msec() + ) + + self._state_cache[frozenset(event_ids)] = cache + + defer.returnValue((None, new_state, prev_states)) + + def resolve_events(self, state_sets, event): + if event.is_state(): + return self._resolve_events( + state_sets, event.type, event.state_key + ) + else: + return self._resolve_events(state_sets) + + def _resolve_events(self, state_sets, event_type=None, state_key=""): state = {} - for group, g_state in state_groups.items(): - for s in g_state: + for st in state_sets: + for e in st: state.setdefault( - (s.type, s.state_key), + (e.type, e.state_key), {} - )[s.event_id] = s + )[e.event_id] = e unconflicted_state = { k: v.values()[0] for k, v in state.items() @@ -302,16 +326,7 @@ class StateHandler(object): new_state = unconflicted_state new_state.update(resolved_state) - if self._state_cache is not None: - cache = _StateCacheEntry( - state=new_state, - state_group=None, - ts=self.clock.time_msec() - ) - - self._state_cache[frozenset(event_ids)] = cache - - defer.returnValue((None, new_state, prev_states)) + return new_state, prev_states @log_function def _resolve_state_events(self, conflicted_state, auth_events): -- cgit 1.4.1 From e441c10a7302a3b0ddfa3caa7823fa6f62372f82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 14:23:39 +0000 Subject: pyflakes --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5b225ffb57..d667d358ab 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -878,7 +878,7 @@ class FederationHandler(BaseHandler): (d.type, d.state_key) for d in different_events }) - new_state, _ = self.state.resolve_events( + new_state, prev_state = self.state.resolve_events( [local_view, remote_view], event ) -- cgit 1.4.1 From b11a6e1c3ca51e81b911fa733a4169bd6cbaa5ee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 15:37:18 +0000 Subject: Fix wrong variable name --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index d667d358ab..4cd795e202 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -878,7 +878,7 @@ class FederationHandler(BaseHandler): (d.type, d.state_key) for d in different_events }) - new_state, prev_state = self.state.resolve_events( + new_state, prev_state = self.state_handler.resolve_events( [local_view, remote_view], event ) -- cgit 1.4.1 From b99a33f283d595f5f462ab039d672ca04bdffcb8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sun, 15 Feb 2015 20:20:51 +0000 Subject: resolve_events expect lists, not dicts --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4cd795e202..3aecc1ca51 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -879,7 +879,7 @@ class FederationHandler(BaseHandler): }) new_state, prev_state = self.state_handler.resolve_events( - [local_view, remote_view], + [local_view.values(), remote_view.values()], event ) -- cgit 1.4.1 From 72a4de2ce627528a13bda480403344ffde6275d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 10:03:23 +0000 Subject: Use consumeErrors=True on all DeferredLists. This is so that the DeferredLists actually consume the error instead of propogating down the non-existent errback chain. This should reduce the number of unhandled errors we are seeing. --- synapse/federation/federation_server.py | 2 +- synapse/federation/transaction_queue.py | 2 +- synapse/handlers/presence.py | 8 ++++---- synapse/notifier.py | 6 ++++-- synapse/storage/roommember.py | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 9f5c98694c..e94d0411b4 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -124,7 +124,7 @@ class FederationServer(FederationBase): edu.content ) - results = yield defer.DeferredList(dl) + results = yield defer.DeferredList(dl, consumeErrors=True) ret = [] for r in results: diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 731019ad9f..bb20f2ebab 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -98,7 +98,7 @@ class TransactionQueue(object): deferreds.append(deferred) - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) # NO inlineCallbacks def enqueue_edu(self, edu): diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 59287010ed..8ef248ecf2 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -492,7 +492,7 @@ class PresenceHandler(BaseHandler): user, domain, remoteusers )) - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) def _start_polling_local(self, user, target_user): target_localpart = target_user.localpart @@ -548,7 +548,7 @@ class PresenceHandler(BaseHandler): self._stop_polling_remote(user, domain, remoteusers) ) - return defer.DeferredList(deferreds) + return defer.DeferredList(deferreds, consumeErrors=True) def _stop_polling_local(self, user, target_user): for localpart in self._local_pushmap.keys(): @@ -729,7 +729,7 @@ class PresenceHandler(BaseHandler): del self._remote_sendmap[user] with PreserveLoggingContext(): - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) @defer.inlineCallbacks def push_update_to_local_and_remote(self, observed_user, statuscache, @@ -768,7 +768,7 @@ class PresenceHandler(BaseHandler): ) ) - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) defer.returnValue((localusers, remote_domains)) diff --git a/synapse/notifier.py b/synapse/notifier.py index e3b6ead620..f5a394596d 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -135,7 +135,8 @@ class Notifier(object): with PreserveLoggingContext(): yield defer.DeferredList( - [notify(l).addErrback(eb) for l in listeners] + [notify(l).addErrback(eb) for l in listeners], + consumeErrors=True, ) @defer.inlineCallbacks @@ -203,7 +204,8 @@ class Notifier(object): with PreserveLoggingContext(): yield defer.DeferredList( - [notify(l).addErrback(eb) for l in listeners] + [notify(l).addErrback(eb) for l in listeners], + consumeErrors=True, ) @defer.inlineCallbacks diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 779f9ce544..9bf608bc90 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -288,7 +288,7 @@ class RoomMemberStore(SQLBaseStore): deferreds = [self.get_rooms_for_user(u) for u in user_id_list] - results = yield defer.DeferredList(deferreds) + results = yield defer.DeferredList(deferreds, consumeErrors=True) # A list of sets of strings giving room IDs for each user room_id_lists = [set([r.room_id for r in result[1]]) for result in results] -- cgit 1.4.1 From c2b1dbd84c27be8759b14502b124504cba4c9295 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 11:11:11 +0000 Subject: We do want to consumeError --- synapse/handlers/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index c69787005f..c2762f92c7 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -181,7 +181,7 @@ class TypingNotificationHandler(BaseHandler): }, )) - yield defer.DeferredList(deferreds, consumeErrors=False) + yield defer.DeferredList(deferreds, consumeErrors=True) @defer.inlineCallbacks def _recv_edu(self, origin, content): -- cgit 1.4.1 From 47281f8fa4b3f529f84409367736483da66416ae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 11:51:22 +0000 Subject: Change some debug logging to info --- synapse/handlers/federation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3aecc1ca51..0eb2ff95ca 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -802,7 +802,7 @@ class FederationHandler(BaseHandler): missing_auth = event_auth_events - seen_events if missing_auth: - logger.debug("Missing auth: %s", missing_auth) + logger.info("Missing auth: %s", missing_auth) # If we don't have all the auth events, we need to get them. try: remote_auth_chain = yield self.replication_layer.get_event_auth( @@ -856,7 +856,7 @@ class FederationHandler(BaseHandler): if different_auth and not event.internal_metadata.is_outlier(): # Do auth conflict res. - logger.debug("Different auth: %s", different_auth) + logger.info("Different auth: %s", different_auth) different_events = yield defer.gatherResults( [ @@ -892,6 +892,8 @@ class FederationHandler(BaseHandler): context.state_group = None if different_auth and not event.internal_metadata.is_outlier(): + logger.info("Different auth after resolution: %s", different_auth) + # Only do auth resolution if we have something new to say. # We can't rove an auth failure. do_resolution = False -- cgit 1.4.1 From 0ac2a79faa918280767c18e4db7ec29d7d3a3afb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 17:24:14 +0000 Subject: Initial stab at implementing a batched get_missing_pdus request --- synapse/federation/federation_server.py | 72 +++++++++++++++++++++++++++++++++ synapse/handlers/federation.py | 9 +++-- synapse/storage/event_federation.py | 63 ++++++++++++++++++++++++++--- 3 files changed, 135 insertions(+), 9 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 22b9663831..34bc397e8a 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -305,6 +305,78 @@ class FederationServer(FederationBase): (200, send_content) ) + @defer.inlineCallbacks + def get_missing_events(self, origin, room_id, earliest_events, + latest_events, limit, min_depth): + limit = max(limit, 50) + min_depth = max(min_depth, 0) + + missing_events = yield self.store.get_missing_events( + room_id=room_id, + earliest_events=earliest_events, + latest_events=latest_events, + limit=limit, + min_depth=min_depth, + ) + + known_ids = {e.event_id for e in missing_events} | {earliest_events} + + back_edges = { + e for e in missing_events + if {i for i, h in e.prev_events.items()} <= known_ids + } + + decoded_auth_events = set() + state = {} + auth_events = set() + auth_and_state = {} + for event in back_edges: + state_pdus = yield self.handler.get_state_for_pdu( + origin, room_id, event.event_id, + do_auth=False, + ) + + state[event.event_id] = [s.event_id for s in state_pdus] + + auth_and_state.update({ + s.event_id: s for s in state_pdus + }) + + state_ids = {pdu.event_id for pdu in state_pdus} + prev_ids = {i for i, h in event.prev_events.items()} + partial_auth_chain = yield self.store.get_auth_chain( + state_ids | prev_ids, have_ids=decoded_auth_events.keys() + ) + + for p in partial_auth_chain: + p.signatures.update( + compute_event_signature( + p, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) + + auth_events.update( + a.event_id for a in partial_auth_chain + ) + + auth_and_state.update({ + a.event_id: a for a in partial_auth_chain + }) + + time_now = self._clock.time_msec() + + defer.returnValue({ + "events": [ev.get_pdu_json(time_now) for ev in missing_events], + "state_for_events": state, + "auth_events": auth_events, + "event_map": { + k: ev.get_pdu_json(time_now) + for k, ev in auth_and_state.items() + }, + }) + @log_function def _get_persisted_pdu(self, origin, event_id, do_auth=True): """ Get a PDU from the database with given origin and id. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0eb2ff95ca..26bdc6d1a7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -581,12 +581,13 @@ class FederationHandler(BaseHandler): defer.returnValue(event) @defer.inlineCallbacks - def get_state_for_pdu(self, origin, room_id, event_id): + def get_state_for_pdu(self, origin, room_id, event_id, do_auth=True): yield run_on_reactor() - in_room = yield self.auth.check_host_in_room(room_id, origin) - if not in_room: - raise AuthError(403, "Host not in room.") + if do_auth: + in_room = yield self.auth.check_host_in_room(room_id, origin) + if not in_room: + raise AuthError(403, "Host not in room.") state_groups = yield self.store.get_state_groups( [event_id] diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 3fbc090224..22bf7ad832 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -32,15 +32,15 @@ class EventFederationStore(SQLBaseStore): and backfilling from another server respectively. """ - def get_auth_chain(self, event_ids): + def get_auth_chain(self, event_ids, have_ids=set()): return self.runInteraction( "get_auth_chain", self._get_auth_chain_txn, - event_ids + event_ids, have_ids ) - def _get_auth_chain_txn(self, txn, event_ids): - results = self._get_auth_chain_ids_txn(txn, event_ids) + def _get_auth_chain_txn(self, txn, event_ids, have_ids): + results = self._get_auth_chain_ids_txn(txn, event_ids, have_ids) return self._get_events_txn(txn, results) @@ -51,8 +51,9 @@ class EventFederationStore(SQLBaseStore): event_ids ) - def _get_auth_chain_ids_txn(self, txn, event_ids): + def _get_auth_chain_ids_txn(self, txn, event_ids, have_ids): results = set() + have_ids = set(have_ids) base_sql = ( "SELECT auth_id FROM event_auth WHERE event_id = ?" @@ -64,6 +65,10 @@ class EventFederationStore(SQLBaseStore): for f in front: txn.execute(base_sql, (f,)) new_front.update([r[0] for r in txn.fetchall()]) + + new_front -= results + new_front -= have_ids + front = new_front results.update(front) @@ -378,3 +383,51 @@ class EventFederationStore(SQLBaseStore): event_results += new_front return self._get_events_txn(txn, event_results) + + def get_missing_events(self, room_id, earliest_events, latest_events, + limit, min_depth): + return self.runInteraction( + "get_missing_events", + self._get_missing_events, + room_id, earliest_events, latest_events, limit, min_depth + ) + + def _get_missing_events(self, txn, room_id, earliest_events, latest_events, + limit, min_depth): + + earliest_events = set(earliest_events) + front = set(latest_events) - earliest_events + + event_results = set() + + query = ( + "SELECT prev_event_id FROM event_edges " + "WHERE room_id = ? AND event_id = ? AND is_state = 0 " + "LIMIT ?" + ) + + while front and len(event_results) < limit: + new_front = set() + for event_id in front: + txn.execute( + query, + (room_id, event_id, limit - len(event_results)) + ) + + for e_id, in txn.fetchall(): + new_front.add(e_id) + + new_front -= earliest_events + new_front -= event_results + + front = new_front + event_results |= new_front + + events = self._get_events_txn(txn, event_results) + + events = sorted( + [ev for ev in events if ev.depth >= min_depth], + key=lambda e: e.depth, + ) + + return events[:limit] -- cgit 1.4.1 From 2bafeca2703635ea1fee2a7617c30cfad1b00315 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 14:08:42 +0000 Subject: Add missing comma so that it generates a dict and not a set --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0eb2ff95ca..7deed16f9c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -875,7 +875,7 @@ class FederationHandler(BaseHandler): local_view = dict(auth_events) remote_view = dict(auth_events) remote_view.update({ - (d.type, d.state_key) for d in different_events + (d.type, d.state_key): d for d in different_events }) new_state, prev_state = self.state_handler.resolve_events( -- cgit 1.4.1 From db215b7e0007a207b8775d78c6693153e16f2731 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Feb 2015 13:58:02 +0000 Subject: Implement and use new batched get missing pdu --- synapse/federation/federation_client.py | 19 ++++ synapse/federation/federation_server.py | 150 +++++++++++--------------------- synapse/federation/transaction_queue.py | 2 +- synapse/federation/transport/client.py | 19 ++++ synapse/federation/transport/server.py | 31 +++++++ synapse/handlers/federation.py | 23 +++++ 6 files changed, 144 insertions(+), 100 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index cd3c962d50..ca89a0787c 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -439,6 +439,25 @@ class FederationClient(FederationBase): defer.returnValue(ret) + @defer.inlineCallbacks + def get_missing_events(self, destination, room_id, earliest_events, + latest_events, limit, min_depth): + content = yield self.transport_layer.get_missing_events( + destination, room_id, earliest_events, latest_events, limit, + min_depth, + ) + + events = [ + self.event_from_pdu_json(e) + for e in content.get("events", []) + ] + + signed_events = yield self._check_sigs_and_hash_and_fetch( + destination, events, outlier=True + ) + + defer.returnValue(signed_events) + def event_from_pdu_json(self, pdu_json, outlier=False): event = FrozenEvent( pdu_json diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 34bc397e8a..f74e16abd5 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -142,7 +142,15 @@ class FederationServer(FederationBase): if r[0]: ret.append({}) else: - logger.exception(r[1]) + failure = r[1] + logger.error( + "Failed to handle PDU", + exc_info=( + failure.type, + failure.value, + failure.getTracebackObject() + ) + ) ret.append({"error": str(r[1].value)}) logger.debug("Returning: %s", str(ret)) @@ -306,75 +314,17 @@ class FederationServer(FederationBase): ) @defer.inlineCallbacks - def get_missing_events(self, origin, room_id, earliest_events, - latest_events, limit, min_depth): - limit = max(limit, 50) - min_depth = max(min_depth, 0) - - missing_events = yield self.store.get_missing_events( - room_id=room_id, - earliest_events=earliest_events, - latest_events=latest_events, - limit=limit, - min_depth=min_depth, + @log_function + def on_get_missing_events(self, origin, room_id, earliest_events, + latest_events, limit, min_depth): + missing_events = yield self.handler.on_get_missing_events( + origin, room_id, earliest_events, latest_events, limit, min_depth ) - known_ids = {e.event_id for e in missing_events} | {earliest_events} - - back_edges = { - e for e in missing_events - if {i for i, h in e.prev_events.items()} <= known_ids - } - - decoded_auth_events = set() - state = {} - auth_events = set() - auth_and_state = {} - for event in back_edges: - state_pdus = yield self.handler.get_state_for_pdu( - origin, room_id, event.event_id, - do_auth=False, - ) - - state[event.event_id] = [s.event_id for s in state_pdus] - - auth_and_state.update({ - s.event_id: s for s in state_pdus - }) - - state_ids = {pdu.event_id for pdu in state_pdus} - prev_ids = {i for i, h in event.prev_events.items()} - partial_auth_chain = yield self.store.get_auth_chain( - state_ids | prev_ids, have_ids=decoded_auth_events.keys() - ) - - for p in partial_auth_chain: - p.signatures.update( - compute_event_signature( - p, - self.hs.hostname, - self.hs.config.signing_key[0] - ) - ) - - auth_events.update( - a.event_id for a in partial_auth_chain - ) - - auth_and_state.update({ - a.event_id: a for a in partial_auth_chain - }) - time_now = self._clock.time_msec() defer.returnValue({ "events": [ev.get_pdu_json(time_now) for ev in missing_events], - "state_for_events": state, - "auth_events": auth_events, - "event_map": { - k: ev.get_pdu_json(time_now) - for k, ev in auth_and_state.items() - }, }) @log_function @@ -403,7 +353,7 @@ class FederationServer(FederationBase): @defer.inlineCallbacks @log_function - def _handle_new_pdu(self, origin, pdu, max_recursion=10): + def _handle_new_pdu(self, origin, pdu, get_missing=True): # We reprocess pdus when we have seen them only as outliers existing = yield self._get_persisted_pdu( origin, pdu.event_id, do_auth=False @@ -455,48 +405,50 @@ class FederationServer(FederationBase): pdu.room_id, min_depth ) + prevs = {e_id for e_id, _ in pdu.prev_events} + seen = set(have_seen.keys()) + if min_depth and pdu.depth < min_depth: # This is so that we don't notify the user about this # message, to work around the fact that some events will # reference really really old events we really don't want to # send to the clients. pdu.internal_metadata.outlier = True - elif min_depth and pdu.depth > min_depth and max_recursion > 0: - for event_id, hashes in pdu.prev_events: - if event_id not in have_seen: - logger.debug( - "_handle_new_pdu requesting pdu %s", - event_id + elif min_depth and pdu.depth > min_depth: + if get_missing and prevs - seen: + latest_tuples = yield self.store.get_latest_events_in_room( + pdu.room_id + ) + + # We add the prev events that we have seen to the latest + # list to ensure the remote server doesn't give them to us + latest = set(e_id for e_id, _, _ in latest_tuples) + latest |= seen + + missing_events = yield self.get_missing_events( + origin, + pdu.room_id, + earliest_events=list(latest), + latest_events=[pdu.event_id], + limit=10, + min_depth=min_depth, + ) + + for e in missing_events: + yield self._handle_new_pdu( + origin, + e, + get_missing=False ) - try: - new_pdu = yield self.federation_client.get_pdu( - [origin, pdu.origin], - event_id=event_id, - ) - - if new_pdu: - yield self._handle_new_pdu( - origin, - new_pdu, - max_recursion=max_recursion-1 - ) - - logger.debug("Processed pdu %s", event_id) - else: - logger.warn("Failed to get PDU %s", event_id) - fetch_state = True - except: - # TODO(erikj): Do some more intelligent retries. - logger.exception("Failed to get PDU") - fetch_state = True - else: - prevs = {e_id for e_id, _ in pdu.prev_events} - seen = set(have_seen.keys()) - if prevs - seen: - fetch_state = True - else: - fetch_state = True + have_seen = yield self.store.have_events( + [ev for ev, _ in pdu.prev_events] + ) + + prevs = {e_id for e_id, _ in pdu.prev_events} + seen = set(have_seen.keys()) + if prevs - seen: + fetch_state = True if fetch_state: # We need to get the state at this event, since we haven't diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 7d30c924d1..8f1acbe590 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -287,7 +287,7 @@ class TransactionQueue(object): code = 200 if response: - for e_id, r in getattr(response, "pdus", {}).items(): + for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warn( "Transaction returned error for %s: %s", diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 8b137e7128..80d03012b7 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -219,3 +219,22 @@ class TransportLayerClient(object): ) defer.returnValue(content) + + @defer.inlineCallbacks + @log_function + def get_missing_events(self, destination, room_id, earliest_events, + latest_events, limit, min_depth): + path = PREFIX + "/get_missing_events/%s" % (room_id,) + + content = yield self.client.post_json( + destination=destination, + path=path, + data={ + "limit": int(limit), + "min_depth": int(min_depth), + "earliest_events": earliest_events, + "latest_events": latest_events, + } + ) + + defer.returnValue(content) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 2ffb37aa18..ad75c8ddb7 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -234,6 +234,7 @@ class TransportLayerServer(object): ) ) ) + self.server.register_path( "POST", re.compile("^" + PREFIX + "/query_auth/([^/]*)/([^/]*)$"), @@ -245,6 +246,17 @@ class TransportLayerServer(object): ) ) + self.server.register_path( + "POST", + re.compile("^" + PREFIX + "/get_missing_events/([^/]*)/?$"), + self._with_authentication( + lambda origin, content, query, room_id: + self._get_missing_events( + origin, content, room_id, + ) + ) + ) + @defer.inlineCallbacks @log_function def _on_send_request(self, origin, content, query, transaction_id): @@ -344,3 +356,22 @@ class TransportLayerServer(object): ) defer.returnValue((200, new_content)) + + @defer.inlineCallbacks + @log_function + def _get_missing_events(self, origin, content, room_id): + limit = int(content.get("limit", 10)) + min_depth = int(content.get("min_depth", 0)) + earliest_events = content.get("earliest_events", []) + latest_events = content.get("latest_events", []) + + content = yield self.request_handler.on_get_missing_events( + origin, + room_id=room_id, + earliest_events=earliest_events, + latest_events=latest_events, + min_depth=min_depth, + limit=limit, + ) + + defer.returnValue((200, content)) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 26bdc6d1a7..628e62f8b1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -789,6 +789,29 @@ class FederationHandler(BaseHandler): defer.returnValue(ret) + @defer.inlineCallbacks + def on_get_missing_events(self, origin, room_id, earliest_events, + latest_events, limit, min_depth): + in_room = yield self.auth.check_host_in_room( + room_id, + origin + ) + if not in_room: + raise AuthError(403, "Host not in room.") + + limit = min(limit, 20) + min_depth = max(min_depth, 0) + + missing_events = yield self.store.get_missing_events( + room_id=room_id, + earliest_events=earliest_events, + latest_events=latest_events, + limit=limit, + min_depth=min_depth, + ) + + defer.returnValue(missing_events) + @defer.inlineCallbacks @log_function def do_auth(self, origin, event, context, auth_events): -- cgit 1.4.1 From 443ba4eecc59e3dbddfaf5b925408b40e3581800 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Feb 2015 15:00:12 +0000 Subject: %s for strings otherwise you end up sending 'u"foo"' --- synapse/handlers/directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 20ab9e269c..11d20a5d2d 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -160,7 +160,7 @@ class DirectoryHandler(BaseHandler): if not room_id: raise SynapseError( 404, - "Room alias %r not found" % (room_alias.to_string(),), + "Room alias %s not found" % (room_alias.to_string(),), Codes.NOT_FOUND ) -- cgit 1.4.1 From 2d20466f9a1349c97d5a3822eb4ee64f19bbdf27 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 25 Feb 2015 15:00:59 +0000 Subject: Add stub functions and work out execution flow to implement AS event stream polling. --- synapse/handlers/events.py | 3 --- synapse/handlers/room.py | 34 +++++++++++++++++++++++++--------- synapse/storage/appservice.py | 19 +++++++++++++++++++ synapse/storage/stream.py | 21 +++++++++++++++++++++ 4 files changed, 65 insertions(+), 12 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 025e7e7e62..8d5f5c8499 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -69,9 +69,6 @@ class EventStreamHandler(BaseHandler): ) self._streams_per_user[auth_user] += 1 - if pagin_config.from_token is None: - pagin_config.from_token = None - rm_handler = self.hs.get_handlers().room_member_handler room_ids = yield rm_handler.get_rooms_for_user(auth_user) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 914742d913..a8b0c95636 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -510,9 +510,16 @@ class RoomMemberHandler(BaseHandler): def get_rooms_for_user(self, user, membership_list=[Membership.JOIN]): """Returns a list of roomids that the user has any of the given membership states in.""" - rooms = yield self.store.get_rooms_for_user_where_membership_is( - user_id=user.to_string(), membership_list=membership_list + + app_service = yield self.store.get_app_service_by_user_id( + user.to_string() ) + if app_service: + rooms = yield self.store.get_app_service_rooms(app_service) + else: + rooms = yield self.store.get_rooms_for_user_where_membership_is( + user_id=user.to_string(), membership_list=membership_list + ) # For some reason the list of events contains duplicates # TODO(paul): work out why because I really don't think it should @@ -559,13 +566,22 @@ class RoomEventSource(object): to_key = yield self.get_current_key() - events, end_key = yield self.store.get_room_events_stream( - user_id=user.to_string(), - from_key=from_key, - to_key=to_key, - room_id=None, - limit=limit, - ) + app_service = self.store.get_app_service_by_user_id(user.to_string()) + if app_service: + events, end_key = yield self.store.get_appservice_room_stream( + service=app_service, + from_key=from_key, + to_key=to_key, + limit=limit, + ) + else: + events, end_key = yield self.store.get_room_events_stream( + user_id=user.to_string(), + from_key=from_key, + to_key=to_key, + room_id=None, + limit=limit, + ) defer.returnValue((events, end_key)) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index dc3666efd4..435ccfd6fc 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -17,6 +17,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError from synapse.appservice import ApplicationService +from synapse.storage.roommember import RoomsForUser from ._base import SQLBaseStore @@ -150,6 +151,16 @@ class ApplicationServiceStore(SQLBaseStore): yield self.cache_defer # make sure the cache is ready defer.returnValue(self.services_cache) + @defer.inlineCallbacks + def get_app_service_by_user_id(self, user_id): + yield self.cache_defer # make sure the cache is ready + + for service in self.services_cache: + if service.sender == user_id: + defer.returnValue(service) + return + defer.returnValue(None) + @defer.inlineCallbacks def get_app_service_by_token(self, token, from_cache=True): """Get the application service with the given token. @@ -173,6 +184,14 @@ class ApplicationServiceStore(SQLBaseStore): # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. + @defer.inlineCallbacks + def get_app_service_rooms(self, service): + logger.info("get_app_service_rooms -> %s", service) + + # TODO stub + yield self.cache_defer + defer.returnValue([RoomsForUser("!foo:bar", service.sender, "join")]) + @defer.inlineCallbacks def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3ccb6f8a61..aa3c9f8c9c 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -127,6 +127,27 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): class StreamStore(SQLBaseStore): + + def get_appservice_room_stream(self, service, from_key, to_key, limit=0): + # NB this lives here instead of appservice.py so we can reuse the + # 'private' StreamToken class in this file. + logger.info("get_appservice_room_stream -> %s", service) + + if limit: + limit = max(limit, MAX_STREAM_SIZE) + else: + limit = MAX_STREAM_SIZE + + # From and to keys should be integers from ordering. + from_id = _StreamToken.parse_stream_token(from_key) + to_id = _StreamToken.parse_stream_token(to_key) + + if from_key == to_key: + return defer.succeed(([], to_key)) + + # TODO stub + return defer.succeed(([], to_key)) + @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, limit=0, with_feedback=False): -- cgit 1.4.1 From dcec7175dc30754603618351b59bc72ff41d305b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 26 Feb 2015 16:23:01 +0000 Subject: Finish impl to get new events for AS. ASes should now be able to poll /events --- synapse/handlers/room.py | 4 ++- synapse/storage/stream.py | 62 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 8 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a8b0c95636..80f7ee3f12 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -566,7 +566,9 @@ class RoomEventSource(object): to_key = yield self.get_current_key() - app_service = self.store.get_app_service_by_user_id(user.to_string()) + app_service = yield self.store.get_app_service_by_user_id( + user.to_string() + ) if app_service: events, end_key = yield self.store.get_appservice_room_stream( service=app_service, diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3c8f3320f1..6946e9fe70 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -128,25 +128,73 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): class StreamStore(SQLBaseStore): + @defer.inlineCallbacks def get_appservice_room_stream(self, service, from_key, to_key, limit=0): # NB this lives here instead of appservice.py so we can reuse the # 'private' StreamToken class in this file. - logger.info("get_appservice_room_stream -> %s", service) - if limit: limit = max(limit, MAX_STREAM_SIZE) else: limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. - # from_id = _StreamToken.parse_stream_token(from_key) - # to_id = _StreamToken.parse_stream_token(to_key) + from_id = _StreamToken.parse_stream_token(from_key) + to_id = _StreamToken.parse_stream_token(to_key) if from_key == to_key: - return defer.succeed(([], to_key)) + defer.returnValue(([], to_key)) + return + + # Logic: + # - We want ALL events which match the AS room_id regex + # - We want ALL events which match the rooms represented by the AS + # room_alias regex + # - We want ALL events for rooms that AS users have joined. + # This is currently supported via get_app_service_rooms (which is used + # for the Notifier listener rooms). We can't reasonably make a SQL + # query for these room IDs, so we'll pull all the events between from/to + # and filter in python. + rooms_for_as = yield self.get_app_service_rooms(service) + room_ids_for_as = [r.room_id for r in rooms_for_as] + + # select all the events between from/to with a sensible limit + sql = ( + "SELECT e.event_id, e.room_id, e.stream_ordering FROM events AS e " + "WHERE e.stream_ordering > ? AND e.stream_ordering <= ? " + "ORDER BY stream_ordering ASC LIMIT %(limit)d " + ) % { + "limit": limit + } + + def f(txn): + txn.execute(sql, (from_id.stream, to_id.stream,)) + + rows = self.cursor_to_dict(txn) + + ret = self._get_events_txn( + txn, + # apply the filter on the room id list + [ + r["event_id"] for r in rows + if r["room_id"] in room_ids_for_as + ], + get_prev_content=True + ) + + self._set_before_and_after(ret, rows) + + if rows: + key = "s%d" % max([r["stream_ordering"] for r in rows]) + + else: + # Assume we didn't get anything because there was nothing to + # get. + key = to_key + + return ret, key - # TODO stub - return defer.succeed(([], to_key)) + results = yield self.runInteraction("get_appservice_room_stream", f) + defer.returnValue(results) @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, -- cgit 1.4.1 From 58ff0660642e6ef8f9fe5672fb50e5d75a569479 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 27 Feb 2015 13:51:41 +0000 Subject: Implement exclusive namespace checks. --- synapse/handlers/directory.py | 14 ++++++++++++-- synapse/handlers/register.py | 11 ++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 11d20a5d2d..f76febee8f 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -232,13 +232,23 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def can_modify_alias(self, alias, user_id=None): + # Any application service "interested" in an alias they are regexing on + # can modify the alias. + # Users can only modify the alias if ALL the interested services have + # non-exclusive locks on the alias (or there are no interested services) services = yield self.store.get_app_services() interested_services = [ s for s in services if s.is_interested_in_alias(alias.to_string()) ] + for service in interested_services: if user_id == service.sender: - # this user IS the app service + # this user IS the app service so they can do whatever they like defer.returnValue(True) return - defer.returnValue(len(interested_services) == 0) + elif service.is_exclusive_alias(alias.to_string()): + # another service has an exclusive lock on this alias. + defer.returnValue(False) + return + # either no interested services, or no service with an exclusive lock + defer.returnValue(True) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 516a936cee..cda4a8502a 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -201,11 +201,12 @@ class RegistrationHandler(BaseHandler): interested_services = [ s for s in services if s.is_interested_in_user(user_id) ] - if len(interested_services) > 0: - raise SynapseError( - 400, "This user ID is reserved by an application service.", - errcode=Codes.EXCLUSIVE - ) + for service in interested_services: + if service.is_exclusive_user(user_id): + raise SynapseError( + 400, "This user ID is reserved by an application service.", + errcode=Codes.EXCLUSIVE + ) def _generate_token(self, user_id): # urlsafe variant uses _ and - so use . as the separator and replace -- cgit 1.4.1 From 130df8fb01bce1bf244940131d85d1c1a54ed5ef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Mar 2015 10:25:36 +0000 Subject: Add some randomness to the user specified timeout on event streams to mitigate against thundering herds problems --- synapse/handlers/events.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'synapse/handlers') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 8d5f5c8499..d3297b7292 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -23,6 +23,7 @@ from synapse.events.utils import serialize_event from ._base import BaseHandler import logging +import random logger = logging.getLogger(__name__) @@ -72,6 +73,14 @@ class EventStreamHandler(BaseHandler): rm_handler = self.hs.get_handlers().room_member_handler room_ids = yield rm_handler.get_rooms_for_user(auth_user) + if timeout: + # If they've set a timeout set a minimum limit. + timeout = max(timeout, 500) + + # Add some randomness to this value to try and mitigate against + # thundering herds on restart. + timeout = random.randint(int(timeout*0.9), int(timeout*1.1)) + with PreserveLoggingContext(): events, tokens = yield self.notifier.get_events_for( auth_user, room_ids, pagin_config, timeout -- cgit 1.4.1 From 9ccccd4874fe827baa2583829602663d1936b0ae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Mar 2015 16:24:05 +0000 Subject: When setting display name more graciously handle failures to update room state. --- synapse/handlers/profile.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'synapse/handlers') diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 03b2159c53..2ddf9d5378 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -212,10 +212,16 @@ class ProfileHandler(BaseHandler): ) msg_handler = self.hs.get_handlers().message_handler - yield msg_handler.create_and_send_event({ - "type": EventTypes.Member, - "room_id": j.room_id, - "state_key": user.to_string(), - "content": content, - "sender": user.to_string() - }, ratelimit=False) + try: + yield msg_handler.create_and_send_event({ + "type": EventTypes.Member, + "room_id": j.room_id, + "state_key": user.to_string(), + "content": content, + "sender": user.to_string() + }, ratelimit=False) + except Exception as e: + logger.warn( + "Failed to update join event for room %s - %s", + j.room_id, str(e.message) + ) -- cgit 1.4.1