From 95037d8d9df113fef85953a6f277b095bda997ad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Sep 2014 16:16:26 +0100 Subject: Change the default power levels to be 0, 50 and 100 --- synapse/api/auth.py | 4 ++-- synapse/handlers/room.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index b4eda3df01..0681a1f710 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -172,7 +172,7 @@ class Auth(object): if kick_level: kick_level = int(kick_level) else: - kick_level = 5 + kick_level = 50 if user_level < kick_level: raise AuthError( @@ -189,7 +189,7 @@ class Auth(object): if ban_level: ban_level = int(ban_level) else: - ban_level = 5 # FIXME (erikj): What should we do here? + ban_level = 50 # FIXME (erikj): What should we do here? if user_level < ban_level: raise AuthError(403, "You don't have permission to ban") diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 8171e9eb45..171ca3d797 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -132,7 +132,7 @@ class RoomCreationHandler(BaseRoomHandler): etype=RoomNameEvent.TYPE, room_id=room_id, user_id=user_id, - required_power_level=5, + required_power_level=50, content={"name": name}, ) @@ -143,7 +143,7 @@ class RoomCreationHandler(BaseRoomHandler): etype=RoomNameEvent.TYPE, room_id=room_id, user_id=user_id, - required_power_level=5, + required_power_level=50, content={"name": name}, ) @@ -155,7 +155,7 @@ class RoomCreationHandler(BaseRoomHandler): etype=RoomTopicEvent.TYPE, room_id=room_id, user_id=user_id, - required_power_level=5, + required_power_level=50, content={"topic": topic}, ) @@ -186,7 +186,7 @@ class RoomCreationHandler(BaseRoomHandler): event_keys = { "room_id": room_id, "user_id": creator.to_string(), - "required_power_level": 10, + "required_power_level": 100, } def create(etype, **content): @@ -203,7 +203,7 @@ class RoomCreationHandler(BaseRoomHandler): power_levels_event = self.event_factory.create_event( etype=RoomPowerLevelsEvent.TYPE, - content={creator.to_string(): 10, "default": 0}, + content={creator.to_string(): 100, "default": 0}, **event_keys ) @@ -215,7 +215,7 @@ class RoomCreationHandler(BaseRoomHandler): add_state_event = create( etype=RoomAddStateLevelEvent.TYPE, - level=10, + level=100, ) send_event = create( @@ -225,8 +225,8 @@ class RoomCreationHandler(BaseRoomHandler): ops = create( etype=RoomOpsPowerLevelsEvent.TYPE, - ban_level=5, - kick_level=5, + ban_level=50, + kick_level=50, ) return [ -- cgit 1.4.1 From 250ee2ea7db628036229a80b8317cd796a097ab2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Sep 2014 16:40:23 +0100 Subject: AUth the contents of power level events --- synapse/api/auth.py | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 0681a1f710..5d7c607702 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.api.constants import Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes -from synapse.api.events.room import RoomMemberEvent +from synapse.api.events.room import RoomMemberEvent, RoomPowerLevelsEvent from synapse.util.logutils import log_function import logging @@ -67,6 +67,9 @@ class Auth(object): else: yield self._can_send_event(event) + if event.type == RoomPowerLevelsEvent.TYPE: + yield self._check_power_levels(event) + defer.returnValue(True) else: raise AuthError(500, "Unknown event: %s" % event) @@ -315,3 +318,71 @@ class Auth(object): 403, "You don't have permission to change that state" ) + + @defer.inlineCallbacks + def _check_power_levels(self, event): + current_state = yield self.store.get_current_state( + event.room_id, + event.type, + event.state_key, + ) + + user_level = yield self.store.get_power_level( + event.room_id, + event.user_id, + ) + + if user_level: + user_level = int(user_level) + else: + user_level = 0 + + old_list = current_state.content + + # FIXME (erikj) + old_people = {k: v for k, v in old_list.items() if k.startswith("@")} + new_people = {k: v for k, v in event.content if k.startswith("@")} + + removed = set(old_people.keys()) - set(new_people.keys()) + added = set(old_people.keys()) - set(new_people.keys()) + same = set(old_people.keys()) & set(new_people.keys()) + + for r in removed: + if int(old_list.content[r]) > user_level: + raise AuthError( + 403, + "You don't have permission to change that state" + ) + + for n in new_people: + if int(event.content[n]) > user_level: + raise AuthError( + 403, + "You don't have permission to change that state" + ) + + for s in same: + if int(event.content[s]) != int(old_list[s]): + if int(old_list[s]) > user_level: + raise AuthError( + 403, + "You don't have permission to change that state" + ) + + if "default" in old_list: + old_default = int(old_list["default"]) + + if old_default > user_level: + raise AuthError( + 403, + "You don't have permission to change that state" + ) + + if "default" in event.content: + new_default = int(event.content["default"]) + + if new_default > user_level: + raise AuthError( + 403, + "You don't have permission to change that state" + ) -- cgit 1.4.1 From 982604fbf2c96c05a98b5787f92c40d600c96c99 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Sep 2014 18:09:17 +0100 Subject: Empty string is not a valid JSON object, so don't return them in HTTP responses. --- synapse/rest/login.py | 2 +- synapse/rest/profile.py | 4 ++-- synapse/rest/room.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/login.py b/synapse/rest/login.py index c7bf901c8e..ba49afcaa7 100644 --- a/synapse/rest/login.py +++ b/synapse/rest/login.py @@ -70,7 +70,7 @@ class LoginFallbackRestServlet(RestServlet): def on_GET(self, request): # TODO(kegan): This should be returning some HTML which is capable of # hitting LoginRestServlet - return (200, "") + return (200, {}) def _parse_json(request): diff --git a/synapse/rest/profile.py b/synapse/rest/profile.py index 2e17f87fa1..dad5a208c7 100644 --- a/synapse/rest/profile.py +++ b/synapse/rest/profile.py @@ -51,7 +51,7 @@ class ProfileDisplaynameRestServlet(RestServlet): yield self.handlers.profile_handler.set_displayname( user, auth_user, new_name) - defer.returnValue((200, "")) + defer.returnValue((200, {})) def on_OPTIONS(self, request, user_id): return (200, {}) @@ -86,7 +86,7 @@ class ProfileAvatarURLRestServlet(RestServlet): yield self.handlers.profile_handler.set_avatar_url( user, auth_user, new_name) - defer.returnValue((200, "")) + defer.returnValue((200, {})) def on_OPTIONS(self, request, user_id): return (200, {}) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 308b447090..cef700c81c 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -154,14 +154,14 @@ class RoomStateEventRestServlet(RestServlet): # membership events are special handler = self.handlers.room_member_handler yield handler.change_membership(event) - defer.returnValue((200, "")) + defer.returnValue((200, {})) else: # store random bits of state msg_handler = self.handlers.message_handler yield msg_handler.store_room_data( event=event ) - defer.returnValue((200, "")) + defer.returnValue((200, {})) # TODO: Needs unit testing for generic events + feedback @@ -249,7 +249,7 @@ class JoinRoomAliasServlet(RestServlet): ) handler = self.handlers.room_member_handler yield handler.change_membership(event) - defer.returnValue((200, "")) + defer.returnValue((200, {})) @defer.inlineCallbacks def on_PUT(self, request, room_identifier, txn_id): @@ -416,7 +416,7 @@ class RoomMembershipRestServlet(RestServlet): ) handler = self.handlers.room_member_handler yield handler.change_membership(event) - defer.returnValue((200, "")) + defer.returnValue((200, {})) @defer.inlineCallbacks def on_PUT(self, request, room_id, membership_action, txn_id): -- cgit 1.4.1 From 9dd4570b68fea123fda216b8fc8625fafc9d8e0a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 5 Sep 2014 21:35:56 +0100 Subject: Generate m.room.aliases event when the HS creates a room alias --- synapse/api/auth.py | 7 ++++++- synapse/api/events/room.py | 7 +++++++ synapse/app/homeserver.py | 2 +- synapse/handlers/_base.py | 3 --- synapse/handlers/directory.py | 38 ++++++++++++++++++++++++++++++++----- synapse/handlers/message.py | 4 ++-- synapse/handlers/room.py | 12 +++++++----- synapse/rest/directory.py | 5 ++++- synapse/storage/directory.py | 7 +++++++ synapse/storage/schema/delta/v3.sql | 27 ++++++++++++++++++++++++++ 10 files changed, 94 insertions(+), 18 deletions(-) create mode 100644 synapse/storage/schema/delta/v3.sql (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 5d7c607702..df61794551 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -327,6 +327,11 @@ class Auth(object): event.state_key, ) + if not current_state: + return + else: + current_state = current_state[0] + user_level = yield self.store.get_power_level( event.room_id, event.user_id, @@ -341,7 +346,7 @@ class Auth(object): # FIXME (erikj) old_people = {k: v for k, v in old_list.items() if k.startswith("@")} - new_people = {k: v for k, v in event.content if k.startswith("@")} + new_people = {k: v for k, v in event.content.items() if k.startswith("@")} removed = set(old_people.keys()) - set(new_people.keys()) added = set(old_people.keys()) - set(new_people.keys()) diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index 33f0f0cb99..3a4dbc58ce 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -173,3 +173,10 @@ class RoomOpsPowerLevelsEvent(SynapseStateEvent): def get_content_template(self): return {} + + +class RoomAliasesEvent(SynapseStateEvent): + TYPE = "m.room.aliases" + + def get_content_template(self): + return {} diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 49cf928cc1..d675d8c8f9 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -57,7 +57,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 2 +SCHEMA_VERSION = 3 class SynapseHomeServer(HomeServer): diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 9989fe8670..de4d23bbb3 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -42,9 +42,6 @@ class BaseHandler(object): retry_after_ms=int(1000*(time_allowed - time_now)), ) - -class BaseRoomHandler(BaseHandler): - @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], extra_users=[]): diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 1b9e831fc0..4ab00a761a 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -19,8 +19,10 @@ from ._base import BaseHandler from synapse.api.errors import SynapseError from synapse.http.client import HttpClient +from synapse.api.events.room import RoomAliasesEvent import logging +import sqlite3 logger = logging.getLogger(__name__) @@ -37,7 +39,8 @@ class DirectoryHandler(BaseHandler): ) @defer.inlineCallbacks - def create_association(self, room_alias, room_id, servers=None): + def create_association(self, user_id, room_alias, room_id, servers=None): + # TODO(erikj): Do auth. if not room_alias.is_mine: @@ -54,12 +57,37 @@ class DirectoryHandler(BaseHandler): if not servers: raise SynapseError(400, "Failed to get server list") - yield self.store.create_room_alias_association( - room_alias, - room_id, - servers + + try: + yield self.store.create_room_alias_association( + room_alias, + room_id, + servers + ) + except sqlite3.IntegrityError: + defer.returnValue("Already exists") + + # TODO: Send the room event. + + aliases = yield self.store.get_aliases_for_room(room_id) + + event = self.event_factory.create_event( + etype=RoomAliasesEvent.TYPE, + state_key=self.hs.hostname, + room_id=room_id, + user_id=user_id, + content={"aliases": aliases}, + ) + + snapshot = yield self.store.snapshot_room( + room_id=room_id, + user_id=user_id, ) + yield self.state_handler.handle_new_event(event, snapshot) + yield self._on_new_room_event(event, snapshot, extra_users=[user_id]) + + @defer.inlineCallbacks def get_association(self, room_alias): room_id = None diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index dad2bbd1a4..87fc04478b 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -19,7 +19,7 @@ from synapse.api.constants import Membership from synapse.api.events.room import RoomTopicEvent from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig -from ._base import BaseRoomHandler +from ._base import BaseHandler import logging @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) -class MessageHandler(BaseRoomHandler): +class MessageHandler(BaseHandler): def __init__(self, hs): super(MessageHandler, self).__init__(hs) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 171ca3d797..3fa12841cf 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -25,14 +25,14 @@ from synapse.api.events.room import ( RoomSendEventLevelEvent, RoomOpsPowerLevelsEvent, RoomNameEvent, ) from synapse.util import stringutils -from ._base import BaseRoomHandler +from ._base import BaseHandler import logging logger = logging.getLogger(__name__) -class RoomCreationHandler(BaseRoomHandler): +class RoomCreationHandler(BaseHandler): @defer.inlineCallbacks def create_room(self, user_id, room_id, config): @@ -105,7 +105,9 @@ class RoomCreationHandler(BaseRoomHandler): ) if room_alias: - yield self.store.create_room_alias_association( + directory_handler = self.hs.get_handlers().directory_handler + yield directory_handler.create_association( + user_id=user_id, room_id=room_id, room_alias=room_alias, servers=[self.hs.hostname], @@ -239,7 +241,7 @@ class RoomCreationHandler(BaseRoomHandler): ] -class RoomMemberHandler(BaseRoomHandler): +class RoomMemberHandler(BaseHandler): # TODO(paul): This handler currently contains a messy conflation of # low-level API that works on UserID objects and so on, and REST-level # API that takes ID strings and returns pagination chunks. These concerns @@ -560,7 +562,7 @@ class RoomMemberHandler(BaseRoomHandler): extra_users=[target_user] ) -class RoomListHandler(BaseRoomHandler): +class RoomListHandler(BaseHandler): @defer.inlineCallbacks def get_public_room_list(self): diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py index 18df7c8d8b..31849246a1 100644 --- a/synapse/rest/directory.py +++ b/synapse/rest/directory.py @@ -45,6 +45,8 @@ class ClientDirectoryServer(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): + user = yield self.auth.get_user_by_req(request) + content = _parse_json(request) if not "room_id" in content: raise SynapseError(400, "Missing room_id key", @@ -69,12 +71,13 @@ class ClientDirectoryServer(RestServlet): try: yield dir_handler.create_association( - room_alias, room_id, servers + user.to_string(), room_alias, room_id, servers ) except SynapseError as e: raise e except: logger.exception("Failed to create association") + raise defer.returnValue((200, {})) diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index bf55449253..540eb4c2c4 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -92,3 +92,10 @@ class DirectoryStore(SQLBaseStore): "server": server, } ) + + def get_aliases_for_room(self, room_id): + return self._simple_select_onecol( + "room_aliases", + {"room_id": room_id}, + "room_alias", + ) diff --git a/synapse/storage/schema/delta/v3.sql b/synapse/storage/schema/delta/v3.sql new file mode 100644 index 0000000000..cade295989 --- /dev/null +++ b/synapse/storage/schema/delta/v3.sql @@ -0,0 +1,27 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +CREATE INDEX IF NOT EXISTS room_aliases_alias ON room_aliases(room_alias); +CREATE INDEX IF NOT EXISTS room_aliases_id ON room_aliases(room_id); + + +CREATE INDEX IF NOT EXISTS room_alias_servers_alias ON room_alias_servers(room_alias); + +DELETE FROM room_aliases WHERE rowid NOT IN (SELECT max(rowid) FROM room_aliases GROUP BY room_alias, room_id); + +CREATE UNIQUE INDEX IF NOT EXISTS room_aliases_uniq ON room_aliases(room_alias, room_id); + +PRAGMA user_version = 3; -- cgit 1.4.1 From 480438eee685c83384d59d8d07477f41e6afad6b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 5 Sep 2014 21:54:16 +0100 Subject: Validate power levels event changes. Change error messages to be more helpful. Fix bug where we checked the wrong power levels --- synapse/api/auth.py | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index df61794551..8f32191b57 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.constants import Membership, JoinRules -from synapse.api.errors import AuthError, StoreError, Codes +from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import RoomMemberEvent, RoomPowerLevelsEvent from synapse.util.logutils import log_function @@ -308,7 +308,9 @@ class Auth(object): else: user_level = 0 - logger.debug("Checking power level for %s, %s", event.user_id, user_level) + logger.debug( + "Checking power level for %s, %s", event.user_id, user_level + ) if current_state and hasattr(current_state, "required_power_level"): req = current_state.required_power_level @@ -321,6 +323,24 @@ class Auth(object): @defer.inlineCallbacks def _check_power_levels(self, event): + for k, v in event.content.items(): + if k == "default": + continue + + # FIXME (erikj): We don't want hsob_Ts in content. + if k == "hsob_ts": + continue + + try: + self.hs.parse_userid(k) + except: + raise SynapseError(400, "Not a valid user_id: %s" % (k,)) + + try: + int(v) + except: + raise SynapseError(400, "Not a valid power level: %s" % (v,)) + current_state = yield self.store.get_current_state( event.room_id, event.type, @@ -346,7 +366,10 @@ class Auth(object): # FIXME (erikj) old_people = {k: v for k, v in old_list.items() if k.startswith("@")} - new_people = {k: v for k, v in event.content.items() if k.startswith("@")} + new_people = { + k: v for k, v in event.content.items() + if k.startswith("@") + } removed = set(old_people.keys()) - set(new_people.keys()) added = set(old_people.keys()) - set(new_people.keys()) @@ -356,22 +379,24 @@ class Auth(object): if int(old_list.content[r]) > user_level: raise AuthError( 403, - "You don't have permission to change that state" + "You don't have permission to remove user: %s" % (r, ) ) - for n in new_people: + for n in added: if int(event.content[n]) > user_level: raise AuthError( 403, - "You don't have permission to change that state" + "You don't have permission to add ops level greater " + "than your own" ) for s in same: if int(event.content[s]) != int(old_list[s]): - if int(old_list[s]) > user_level: + if int(event.content[s]) > user_level: raise AuthError( 403, - "You don't have permission to change that state" + "You don't have permission to add ops level greater " + "than your own" ) if "default" in old_list: @@ -380,7 +405,8 @@ class Auth(object): if old_default > user_level: raise AuthError( 403, - "You don't have permission to change that state" + "You don't have permission to add ops level greater than " + "your own" ) if "default" in event.content: @@ -389,5 +415,6 @@ class Auth(object): if new_default > user_level: raise AuthError( 403, - "You don't have permission to change that state" + "You don't have permission to add ops level greater " + "than your own" ) -- cgit 1.4.1 From f47f42090d58ec5c49b3e14c50a62e96744980ca Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 6 Sep 2014 01:10:07 +0100 Subject: Add support for inviting people when you create a room --- synapse/handlers/room.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'synapse') diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 3fa12841cf..a0d0f2af16 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -65,6 +65,13 @@ class RoomCreationHandler(BaseHandler): else: room_alias = None + invite_list = config.get("invite", []) + for i in invite_list: + try: + self.hs.parse_userid(i) + except: + raise SynapseError(400, "Invalid user_id: %s" % (i,)) + is_public = config.get("visibility", None) == "public" if room_id: @@ -178,6 +185,25 @@ class RoomCreationHandler(BaseHandler): do_auth=False ) + content = {"membership": Membership.INVITE} + for invitee in invite_list: + invite_event = self.event_factory.create_event( + etype=RoomMemberEvent.TYPE, + state_key=invitee, + room_id=room_id, + user_id=user_id, + content=content + ) + + yield self.hs.get_handlers().room_member_handler.change_membership( + invite_event, + do_auth=False + ) + + yield self.hs.get_handlers().room_member_handler.change_membership( + join_event, + do_auth=False + ) result = {"room_id": room_id} if room_alias: result["room_alias"] = room_alias.to_string() -- cgit 1.4.1 From 0b9e1e7b562c3b278873060ca3c4109bc2e451e8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 5 Sep 2014 17:58:06 -0700 Subject: Added a captcha config to the HS, to enable registration captcha checking and for the recaptcha private key. --- synapse/api/errors.py | 1 + synapse/config/captcha.py | 36 +++++++++++++++++++++++++++ synapse/config/homeserver.py | 3 ++- synapse/rest/register.py | 6 ++++- webclient/components/matrix/matrix-service.js | 1 - 5 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 synapse/config/captcha.py (limited to 'synapse') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 84afe4fa37..8e9dd2aba6 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -29,6 +29,7 @@ class Codes(object): NOT_FOUND = "M_NOT_FOUND" UNKNOWN_TOKEN = "M_UNKNOWN_TOKEN" LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" + NEEDS_CAPTCHA = "M_NEEDS_CAPTCHA" class CodeMessageException(Exception): diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py new file mode 100644 index 0000000000..021da5c69b --- /dev/null +++ b/synapse/config/captcha.py @@ -0,0 +1,36 @@ +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + +class CaptchaConfig(Config): + + def __init__(self, args): + super(CaptchaConfig, self).__init__(args) + self.recaptcha_private_key = args.recaptcha_private_key + self.enable_registration_captcha = args.enable_registration_captcha + + @classmethod + def add_arguments(cls, parser): + super(CaptchaConfig, cls).add_arguments(parser) + group = parser.add_argument_group("recaptcha") + group.add_argument( + "--recaptcha-private-key", type=str, default="YOUR_PRIVATE_KEY", + help="The matching private key for the web client's public key." + ) + group.add_argument( + "--enable-registration-captcha", type=bool, default=False, + help="Enables ReCaptcha checks when registering, preventing signup "+ + "unless a captcha is answered. Requires a valid ReCaptcha public/private key." + ) \ No newline at end of file diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 76e2cdeddd..e16f2c733b 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -19,9 +19,10 @@ from .logger import LoggingConfig from .database import DatabaseConfig from .ratelimiting import RatelimitConfig from .repository import ContentRepositoryConfig +from .captcha import CaptchaConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, - RatelimitConfig, ContentRepositoryConfig): + RatelimitConfig, ContentRepositoryConfig, CaptchaConfig): pass if __name__=='__main__': diff --git a/synapse/rest/register.py b/synapse/rest/register.py index b8de3b250d..33a80b7a77 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -16,7 +16,7 @@ """This module contains REST servlets to do with registration: /register""" from twisted.internet import defer -from synapse.api.errors import SynapseError +from synapse.api.errors import SynapseError, Codes from base import RestServlet, client_path_pattern import json @@ -50,6 +50,10 @@ class RegisterRestServlet(RestServlet): threepidCreds = None if 'threepidCreds' in register_json: threepidCreds = register_json['threepidCreds'] + + if self.hs.config.enable_registration_captcha: + if not "challenge" in register_json or not "response" in register_json: + raise SynapseError(400, "Captcha response is required", errcode=Codes.NEEDS_CAPTCHA) handler = self.handlers.registration_handler (user_id, token) = yield handler.register( diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js index 4754dc87da..cc785269a1 100644 --- a/webclient/components/matrix/matrix-service.js +++ b/webclient/components/matrix/matrix-service.js @@ -107,7 +107,6 @@ angular.module('matrixService', []) challenge: challengeToken, response: captchaEntry }; - console.log("Sending Captcha info: " + JSON.stringify(data.captcha)); } return doRequest("POST", path, undefined, data); -- cgit 1.4.1 From 781ff713ba65e9a8075063ee5112479afcb412ed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 6 Sep 2014 02:23:36 +0100 Subject: When getting a state event also include the previous content --- synapse/api/events/__init__.py | 7 ++++++- synapse/storage/__init__.py | 9 +++++++-- synapse/storage/_base.py | 19 +++++++++++++++++++ synapse/storage/roommember.py | 4 ++-- synapse/storage/stream.py | 17 +++++++++-------- 5 files changed, 43 insertions(+), 13 deletions(-) (limited to 'synapse') diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index f95468fc65..5f300de108 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -157,7 +157,12 @@ class SynapseEvent(JsonEncodedObject): class SynapseStateEvent(SynapseEvent): - def __init__(self, **kwargs): + + valid_keys = SynapseEvent.valid_keys + [ + "prev_content", + ] + + def __init__(self, **kwargs): if "state_key" not in kwargs: kwargs["state_key"] = "" super(SynapseStateEvent, self).__init__(**kwargs) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d97014f4da..81c3c94b2e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -81,7 +81,7 @@ class DataStore(RoomMemberStore, RoomStore, defer.returnValue(latest) @defer.inlineCallbacks - def get_event(self, event_id): + def get_event(self, event_id, allow_none=False): events_dict = yield self._simple_select_one( "events", {"event_id": event_id}, @@ -92,8 +92,12 @@ class DataStore(RoomMemberStore, RoomStore, "content", "unrecognized_keys" ], + allow_none=allow_none, ) + if not events_dict: + defer.returnValue(None) + event = self._parse_event_from_row(events_dict) defer.returnValue(event) @@ -220,7 +224,8 @@ class DataStore(RoomMemberStore, RoomStore, results = yield self._execute_and_decode(sql, *args) - defer.returnValue([self._parse_event_from_row(r) for r in results]) + events = yield self._parse_events(results) + defer.returnValue(events) @defer.inlineCallbacks def _get_min_token(self): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index bae50e7d1f..8037225079 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -312,6 +312,25 @@ class SQLBaseStore(object): **d ) + def _parse_events(self, rows): + return self._db_pool.runInteraction(self._parse_events_txn, rows) + + def _parse_events_txn(self, txn, rows): + events = [self._parse_event_from_row(r) for r in rows] + + sql = "SELECT * FROM events WHERE event_id = ?" + + for ev in events: + if hasattr(ev, "prev_state"): + # Load previous state_content. + # TODO: Should we be pulling this out above? + cursor = txn.execute(sql, (ev.prev_state,)) + prevs = self.cursor_to_dict(cursor) + if prevs: + prev = self._parse_event_from_row(prevs[0]) + ev.prev_content = prev.content + + return events class Table(object): """ A base class used to store information about a particular table. diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 75c9a60101..9a393e2568 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -88,7 +88,7 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, (user_id, room_id)) rows = self.cursor_to_dict(txn) if rows: - return self._parse_event_from_row(rows[0]) + return self._parse_events_txn(txn, rows)[0] else: return None @@ -161,7 +161,7 @@ class RoomMemberStore(SQLBaseStore): # logger.debug("_get_members_query Got rows %s", rows) - results = [self._parse_event_from_row(r) for r in rows] + results = yield self._parse_events(rows) defer.returnValue(results) @defer.inlineCallbacks diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 2cb0067a67..aff6dc9855 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -188,7 +188,7 @@ class StreamStore(SQLBaseStore): user_id, user_id, from_id, to_id ) - ret = [self._parse_event_from_row(r) for r in rows] + ret = yield self._parse_events(rows) if rows: key = "s%d" % max([r["stream_ordering"] for r in rows]) @@ -243,9 +243,11 @@ class StreamStore(SQLBaseStore): # TODO (erikj): We should work out what to do here instead. next_token = to_key if to_key else from_key + events = yield self._parse_events(rows) + defer.returnValue( ( - [self._parse_event_from_row(r) for r in rows], + events, next_token ) ) @@ -277,12 +279,11 @@ class StreamStore(SQLBaseStore): else: token = (end_token, end_token) - defer.returnValue( - ( - [self._parse_event_from_row(r) for r in rows], - token - ) - ) + events = yield self._parse_events(rows) + + ret = (events, token) + + defer.returnValue(ret) def get_room_events_max_id(self): return self._db_pool.runInteraction(self._get_room_events_max_id_txn) -- cgit 1.4.1 From 6d19fe1481820331355669ea35f46bcf6dbfd93b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 6 Sep 2014 02:48:13 +0100 Subject: Fix generation of event ids so that they are consistent between local and remote ids --- synapse/api/events/factory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index a3b293e024..5e38cdbc44 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -47,11 +47,14 @@ class EventFactory(object): self._event_list[event_class.TYPE] = event_class self.clock = hs.get_clock() + self.hs = hs def create_event(self, etype=None, **kwargs): kwargs["type"] = etype if "event_id" not in kwargs: - kwargs["event_id"] = random_string(10) + kwargs["event_id"] = "%s@%s" % ( + random_string(10), self.hs.hostname + ) if "ts" not in kwargs: kwargs["ts"] = int(self.clock.time_msec()) -- cgit 1.4.1 From 1829b55bb0d75d29475ac84eeb3e37cad8b334c7 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 5 Sep 2014 19:18:23 -0700 Subject: Captchas now work on registration. Missing x-forwarded-for config arg support. Missing reloading a new captcha on the web client / displaying a sensible error message. --- synapse/api/errors.py | 16 ++++++++++++++- synapse/handlers/register.py | 49 ++++++++++++++++++++++++++++++++++++++++++-- synapse/http/client.py | 28 ++++++++++++++++++++++++- synapse/rest/register.py | 29 +++++++++++++++++++++++--- 4 files changed, 115 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 8e9dd2aba6..88175602c4 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -29,7 +29,8 @@ class Codes(object): NOT_FOUND = "M_NOT_FOUND" UNKNOWN_TOKEN = "M_UNKNOWN_TOKEN" LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" - NEEDS_CAPTCHA = "M_NEEDS_CAPTCHA" + CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" + CAPTCHA_INVALID = "M_CAPTCHA_INVALID" class CodeMessageException(Exception): @@ -102,6 +103,19 @@ class StoreError(SynapseError): pass +class InvalidCaptchaError(SynapseError): + def __init__(self, code=400, msg="Invalid captcha.", error_url=None, + errcode=Codes.CAPTCHA_INVALID): + super(InvalidCaptchaError, self).__init__(code, msg, errcode) + self.error_url = error_url + + def error_dict(self): + return cs_error( + self.msg, + self.errcode, + error_url=self.error_url, + ) + class LimitExceededError(SynapseError): """A client has sent too many requests and is being throttled. """ diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index bee052274f..cf20b4efd3 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -17,7 +17,7 @@ from twisted.internet import defer from synapse.types import UserID -from synapse.api.errors import SynapseError, RegistrationError +from synapse.api.errors import SynapseError, RegistrationError, InvalidCaptchaError from ._base import BaseHandler import synapse.util.stringutils as stringutils from synapse.http.client import PlainHttpClient @@ -38,7 +38,7 @@ class RegistrationHandler(BaseHandler): self.distributor.declare("registered_user") @defer.inlineCallbacks - def register(self, localpart=None, password=None, threepidCreds=None): + def register(self, localpart=None, password=None, threepidCreds=None, captcha_info={}): """Registers a new client on the server. Args: @@ -51,6 +51,19 @@ class RegistrationHandler(BaseHandler): Raises: RegistrationError if there was a problem registering. """ + if captcha_info: + captcha_response = yield self._validate_captcha( + captcha_info["ip"], + captcha_info["private_key"], + captcha_info["challenge"], + captcha_info["response"] + ) + if not captcha_response["valid"]: + raise InvalidCaptchaError( + error_url=captcha_response["error_url"] + ) + else: + logger.info("Valid captcha entered from %s", captcha_info["ip"]) if threepidCreds: for c in threepidCreds: @@ -153,5 +166,37 @@ class RegistrationHandler(BaseHandler): ) defer.returnValue(data) + @defer.inlineCallbacks + def _validate_captcha(self, ip_addr, private_key, challenge, response): + """Validates the captcha provided. + + Returns: + dict: Containing 'valid'(bool) and 'error_url'(str) if invalid. + + """ + response = yield self._submit_captcha(ip_addr, private_key, challenge, response) + # parse Google's response. Lovely format.. + lines = response.split('\n') + json = { + "valid": lines[0] == 'true', + "error_url": "http://www.google.com/recaptcha/api/challenge?error=%s" % lines[1] + } + defer.returnValue(json) + + @defer.inlineCallbacks + def _submit_captcha(self, ip_addr, private_key, challenge, response): + client = PlainHttpClient(self.hs) + data = yield client.post_urlencoded_get_raw( + "www.google.com:80", + "/recaptcha/api/verify", + accept_partial=True, # twisted dislikes google's response, no content length. + args={ + 'privatekey': private_key, + 'remoteip': ip_addr, + 'challenge': challenge, + 'response': response + } + ) + defer.returnValue(data) diff --git a/synapse/http/client.py b/synapse/http/client.py index ebf1aa47c4..ece6318e00 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -16,7 +16,7 @@ from twisted.internet import defer, reactor from twisted.internet.error import DNSLookupError -from twisted.web.client import _AgentBase, _URI, readBody, FileBodyProducer +from twisted.web.client import _AgentBase, _URI, readBody, FileBodyProducer, PartialDownloadError from twisted.web.http_headers import Headers from synapse.http.endpoint import matrix_endpoint @@ -188,6 +188,32 @@ class TwistedHttpClient(HttpClient): body = yield readBody(response) defer.returnValue(json.loads(body)) + + # XXX FIXME : I'm so sorry. + @defer.inlineCallbacks + def post_urlencoded_get_raw(self, destination, path, accept_partial=False, args={}): + if destination in _destination_mappings: + destination = _destination_mappings[destination] + + query_bytes = urllib.urlencode(args, True) + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + producer=FileBodyProducer(StringIO(urllib.urlencode(args))), + headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]} + ) + + try: + body = yield readBody(response) + defer.returnValue(body) + except PartialDownloadError as e: + if accept_partial: + defer.returnValue(e.response) + else: + raise e + @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, param_bytes=b"", diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 33a80b7a77..3c8929cf9b 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -51,15 +51,38 @@ class RegisterRestServlet(RestServlet): if 'threepidCreds' in register_json: threepidCreds = register_json['threepidCreds'] + captcha = {} if self.hs.config.enable_registration_captcha: - if not "challenge" in register_json or not "response" in register_json: - raise SynapseError(400, "Captcha response is required", errcode=Codes.NEEDS_CAPTCHA) + challenge = None + user_response = None + try: + captcha_type = register_json["captcha"]["type"] + if captcha_type != "m.login.recaptcha": + raise SynapseError(400, "Sorry, only m.login.recaptcha requests are supported.") + challenge = register_json["captcha"]["challenge"] + user_response = register_json["captcha"]["response"] + except KeyError: + raise SynapseError(400, "Captcha response is required", errcode=Codes.CAPTCHA_NEEDED) + + # TODO determine the source IP : May be an X-Forwarding-For header depending on config + ip_addr = request.getClientIP() + #if self.hs.config.captcha_ip_origin_is_x_forwarded: + # # use the header + + captcha = { + "ip": ip_addr, + "private_key": self.hs.config.recaptcha_private_key, + "challenge": challenge, + "response": user_response + } + handler = self.handlers.registration_handler (user_id, token) = yield handler.register( localpart=desired_user_id, password=password, - threepidCreds=threepidCreds) + threepidCreds=threepidCreds, + captcha_info=captcha) result = { "user_id": user_id, -- cgit 1.4.1 From 37e53513b6789b4f9f845a26b64933f1c533ed62 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 5 Sep 2014 22:51:11 -0700 Subject: Add config opion for XFF headers when performing ReCaptcha auth. --- synapse/config/captcha.py | 6 ++++++ synapse/handlers/register.py | 1 + synapse/rest/register.py | 7 +++++-- 3 files changed, 12 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 021da5c69b..a97a5bab1e 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -20,6 +20,7 @@ class CaptchaConfig(Config): super(CaptchaConfig, self).__init__(args) self.recaptcha_private_key = args.recaptcha_private_key self.enable_registration_captcha = args.enable_registration_captcha + self.captcha_ip_origin_is_x_forwarded = args.captcha_ip_origin_is_x_forwarded @classmethod def add_arguments(cls, parser): @@ -33,4 +34,9 @@ class CaptchaConfig(Config): "--enable-registration-captcha", type=bool, default=False, help="Enables ReCaptcha checks when registering, preventing signup "+ "unless a captcha is answered. Requires a valid ReCaptcha public/private key." + ) + group.add_argument( + "--captcha_ip_origin_is_x_forwarded", type=bool, default=False, + help="When checking captchas, use the X-Forwarded-For (XFF) header as the client IP "+ + "and not the actual client IP." ) \ No newline at end of file diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index cf20b4efd3..6b55775de0 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -59,6 +59,7 @@ class RegistrationHandler(BaseHandler): captcha_info["response"] ) if not captcha_response["valid"]: + logger.info("Invalid captcha entered from %s", captcha_info["ip"]) raise InvalidCaptchaError( error_url=captcha_response["error_url"] ) diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 3c8929cf9b..5872a11d80 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -66,8 +66,11 @@ class RegisterRestServlet(RestServlet): # TODO determine the source IP : May be an X-Forwarding-For header depending on config ip_addr = request.getClientIP() - #if self.hs.config.captcha_ip_origin_is_x_forwarded: - # # use the header + if self.hs.config.captcha_ip_origin_is_x_forwarded: + # use the header + if request.requestHeaders.hasHeader("X-Forwarded-For"): + ip_addr = request.requestHeaders.getRawHeaders( + "X-Forwarded-For")[0] captcha = { "ip": ip_addr, -- cgit 1.4.1 From 3ea6f01b4eff682c2770236cd7cee61a7bc61276 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 5 Sep 2014 22:55:29 -0700 Subject: 80 chars please --- synapse/handlers/register.py | 28 +++++++++++++++++++--------- synapse/rest/register.py | 6 ++++-- 2 files changed, 23 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 6b55775de0..0693112ba8 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -17,7 +17,9 @@ from twisted.internet import defer from synapse.types import UserID -from synapse.api.errors import SynapseError, RegistrationError, InvalidCaptchaError +from synapse.api.errors import ( + SynapseError, RegistrationError, InvalidCaptchaError +) from ._base import BaseHandler import synapse.util.stringutils as stringutils from synapse.http.client import PlainHttpClient @@ -38,7 +40,8 @@ class RegistrationHandler(BaseHandler): self.distributor.declare("registered_user") @defer.inlineCallbacks - def register(self, localpart=None, password=None, threepidCreds=None, captcha_info={}): + def register(self, localpart=None, password=None, threepidCreds=None, + captcha_info={}): """Registers a new client on the server. Args: @@ -59,7 +62,8 @@ class RegistrationHandler(BaseHandler): captcha_info["response"] ) if not captcha_response["valid"]: - logger.info("Invalid captcha entered from %s", captcha_info["ip"]) + logger.info("Invalid captcha entered from %s", + captcha_info["ip"]) raise InvalidCaptchaError( error_url=captcha_response["error_url"] ) @@ -68,7 +72,8 @@ class RegistrationHandler(BaseHandler): if threepidCreds: for c in threepidCreds: - logger.info("validating theeepidcred sid %s on id server %s", c['sid'], c['idServer']) + logger.info("validating theeepidcred sid %s on id server %s", + c['sid'], c['idServer']) try: threepid = yield self._threepid_from_creds(c) except: @@ -77,7 +82,8 @@ class RegistrationHandler(BaseHandler): if not threepid: raise RegistrationError(400, "Couldn't validate 3pid") - logger.info("got threepid medium %s address %s", threepid['medium'], threepid['address']) + logger.info("got threepid medium %s address %s", + threepid['medium'], threepid['address']) password_hash = None if password: @@ -145,7 +151,8 @@ class RegistrationHandler(BaseHandler): # XXX: make this configurable! trustedIdServers = [ 'matrix.org:8090' ] if not creds['idServer'] in trustedIdServers: - logger.warn('%s is not a trusted ID server: rejecting 3pid credentials', creds['idServer']) + logger.warn('%s is not a trusted ID server: rejecting 3pid '+ + 'credentials', creds['idServer']) defer.returnValue(None) data = yield httpCli.get_json( creds['idServer'], @@ -163,7 +170,8 @@ class RegistrationHandler(BaseHandler): data = yield httpCli.post_urlencoded_get_json( creds['idServer'], "/_matrix/identity/api/v1/3pid/bind", - { 'sid': creds['sid'], 'clientSecret': creds['clientSecret'], 'mxid':mxid } + { 'sid': creds['sid'], 'clientSecret': creds['clientSecret'], + 'mxid':mxid } ) defer.returnValue(data) @@ -175,12 +183,14 @@ class RegistrationHandler(BaseHandler): dict: Containing 'valid'(bool) and 'error_url'(str) if invalid. """ - response = yield self._submit_captcha(ip_addr, private_key, challenge, response) + response = yield self._submit_captcha(ip_addr, private_key, challenge, + response) # parse Google's response. Lovely format.. lines = response.split('\n') json = { "valid": lines[0] == 'true', - "error_url": "http://www.google.com/recaptcha/api/challenge?error=%s" % lines[1] + "error_url": "http://www.google.com/recaptcha/api/challenge?"+ + "error=%s" % lines[1] } defer.returnValue(json) diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 5872a11d80..48d3c6eca0 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -58,11 +58,13 @@ class RegisterRestServlet(RestServlet): try: captcha_type = register_json["captcha"]["type"] if captcha_type != "m.login.recaptcha": - raise SynapseError(400, "Sorry, only m.login.recaptcha requests are supported.") + raise SynapseError(400, "Sorry, only m.login.recaptcha " + + "requests are supported.") challenge = register_json["captcha"]["challenge"] user_response = register_json["captcha"]["response"] except KeyError: - raise SynapseError(400, "Captcha response is required", errcode=Codes.CAPTCHA_NEEDED) + raise SynapseError(400, "Captcha response is required", + errcode=Codes.CAPTCHA_NEEDED) # TODO determine the source IP : May be an X-Forwarding-For header depending on config ip_addr = request.getClientIP() -- cgit 1.4.1 From b5749c75d90247ff2f7960fad909b7b4fb694b67 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 5 Sep 2014 23:08:39 -0700 Subject: Reload captchas when they fail. Cleanup on success. --- synapse/handlers/register.py | 4 ++-- webclient/login/register-controller.js | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 0693112ba8..0b841d6d3a 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -62,8 +62,8 @@ class RegistrationHandler(BaseHandler): captcha_info["response"] ) if not captcha_response["valid"]: - logger.info("Invalid captcha entered from %s", - captcha_info["ip"]) + logger.info("Invalid captcha entered from %s. Error: %s", + captcha_info["ip"], captcha_response["error_url"]) raise InvalidCaptchaError( error_url=captcha_response["error_url"] ) diff --git a/webclient/login/register-controller.js b/webclient/login/register-controller.js index 96fffb364d..1ab50888df 100644 --- a/webclient/login/register-controller.js +++ b/webclient/login/register-controller.js @@ -92,6 +92,9 @@ angular.module('RegisterController', ['matrixService']) matrixService.register(mxid, password, threepidCreds, useCaptcha).then( function(response) { $scope.feedback = "Success"; + if (useCaptcha) { + Recaptcha.destroy(); + } // Update the current config var config = matrixService.config(); angular.extend(config, { @@ -118,11 +121,17 @@ angular.module('RegisterController', ['matrixService']) }, function(error) { console.trace("Registration error: "+error); + if (useCaptcha) { + Recaptcha.reload(); + } if (error.data) { if (error.data.errcode === "M_USER_IN_USE") { $scope.feedback = "Username already taken."; $scope.reenter_username = true; } + else if (error.data.errcode == "M_CAPTCHA_INVALID") { + $scope.feedback = "Failed captcha."; + } } else if (error.status === 0) { $scope.feedback = "Unable to talk to the server."; -- cgit 1.4.1 From 2205aba3ed1337a00f9f7869db8a164b65b68f81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 6 Sep 2014 07:41:36 +0100 Subject: Fix bug where we used an event_id as a pdu_id --- synapse/state.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 36d8210eb5..5dcff27367 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -16,7 +16,7 @@ from twisted.internet import defer -from synapse.federation.pdu_codec import encode_event_id +from synapse.federation.pdu_codec import encode_event_id, decode_event_id from synapse.util.logutils import log_function from collections import namedtuple @@ -87,9 +87,11 @@ class StateHandler(object): # than the power level of the user # power_level = self._get_power_level_for_event(event) + pdu_id, origin = decode_event_id(event.event_id, self.server_name) + yield self.store.update_current_state( - pdu_id=event.event_id, - origin=self.server_name, + pdu_id=pdu_id, + origin=origin, context=key.context, pdu_type=key.type, state_key=key.state_key -- cgit 1.4.1 From 7735aad9d67e6e20ad61977e54c9ff2447623804 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 6 Sep 2014 17:27:42 +0100 Subject: Bump version and changelog --- CHANGES.rst | 21 +++++++++++++++++++++ VERSION | 2 +- synapse/__init__.py | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/CHANGES.rst b/CHANGES.rst index 31eee891da..8824ece5a9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,24 @@ +Changes in synapse 0.2.2 (2014-09-06) +===================================== + +Homeserver: + * When the server returns state events it now also includes the previous + content. + * Add support for inviting people when creating a new room. + * Make the homeserver inform the room via `m.room.aliases` when a new alias + is added for a room. + * Validate `m.room.power_level` events. + +Webclient: + * Add support for captchas on registration. + * Handle `m.room.aliases` events. + * Asynchronously send messages and show a local echo. + * Inform the UI when a message failed to send. + * Only autoscroll on receiving a new message if the user was already at the + bottom of the screen. + * Add support for ban/kick reasons. + * Fix bug where we occaisonally saw duplicated join messages. + Changes in synapse 0.2.1 (2014-09-03) ===================================== diff --git a/VERSION b/VERSION index 0c62199f16..ee1372d33a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.1 +0.2.2 diff --git a/synapse/__init__.py b/synapse/__init__.py index 440e633966..1ed9cdcdf3 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.2.1" +__version__ = "0.2.2" -- cgit 1.4.1 From 768ff1a850a74141c67f643e46b26884cd149837 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 6 Sep 2014 17:38:11 +0100 Subject: Fix race in presence handler where we evicted things from cache while handling a key therein --- synapse/handlers/presence.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index c79bb6ff76..b2af09f090 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -796,11 +796,12 @@ class PresenceEventSource(object): updates = [] # TODO(paul): use a DeferredList ? How to limit concurrency. for observed_user in cachemap.keys(): - if not (from_key < cachemap[observed_user].serial): + cached = cachemap[observed_user] + if not (from_key < cached.serial): continue if (yield self.is_visible(observer_user, observed_user)): - updates.append((observed_user, cachemap[observed_user])) + updates.append((observed_user, cached)) # TODO(paul): limit -- cgit 1.4.1 From c0577ea87a19c169d68ed83760582fa1fabe36e5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Sep 2014 18:34:18 +0100 Subject: Rollback if we try and insert duplicate events --- synapse/storage/__init__.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 81c3c94b2e..8ed80109a5 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -179,6 +179,7 @@ class DataStore(RoomMemberStore, RoomStore, "Failed to persist, probably duplicate: %s", event.event_id ) + txn.rollback() return if not backfilled and hasattr(event, "state_key"): -- cgit 1.4.1 From 83ce57302dab6a825f3afde11926b5404ce1c9ff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Sep 2014 19:50:46 +0100 Subject: Fix bug in state handling where we incorrectly identified a missing pdu. Update tests to catch this case. --- synapse/state.py | 92 +++++++++---------- synapse/storage/pdu.py | 9 +- tests/test_state.py | 233 +++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 267 insertions(+), 67 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 5dcff27367..e69282860a 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -134,7 +134,9 @@ class StateHandler(object): @defer.inlineCallbacks @log_function def _handle_new_state(self, new_pdu): - tree = yield self.store.get_unresolved_state_tree(new_pdu) + tree, missing_branch = yield self.store.get_unresolved_state_tree( + new_pdu + ) new_branch, current_branch = tree logger.debug( @@ -142,6 +144,28 @@ class StateHandler(object): new_branch, current_branch ) + if missing_branch is not None: + # We're missing some PDUs. Fetch them. + # TODO (erikj): Limit this. + missing_prev = tree[missing_branch][-1] + + pdu_id = missing_prev.prev_state_id + origin = missing_prev.prev_state_origin + + is_missing = yield self.store.get_pdu(pdu_id, origin) is None + if not is_missing: + raise Exception("Conflict resolution failed") + + yield self._replication.get_pdu( + destination=missing_prev.origin, + pdu_origin=origin, + pdu_id=pdu_id, + outlier=True + ) + + updated_current = yield self._handle_new_state(new_pdu) + defer.returnValue(updated_current) + if not current_branch: # There is no current state defer.returnValue(True) @@ -151,65 +175,35 @@ class StateHandler(object): c = current_branch[-1] if n.pdu_id == c.pdu_id and n.origin == c.origin: - # We have all the PDUs we need, so we can just do the conflict - # resolution. + # We found a common ancestor! if len(current_branch) == 1: # This is a direct clobber so we can just... defer.returnValue(True) - conflict_res = [ - self._do_power_level_conflict_res, - self._do_chain_length_conflict_res, - self._do_hash_conflict_res, - ] - - for algo in conflict_res: - new_res, curr_res = algo(new_branch, current_branch) - - if new_res < curr_res: - defer.returnValue(False) - elif new_res > curr_res: - defer.returnValue(True) - - raise Exception("Conflict resolution failed.") - else: - # We need to ask for PDUs. - missing_prev = max( - new_branch[-1], current_branch[-1], - key=lambda x: x.depth - ) - - if not hasattr(missing_prev, "prev_state_id"): - # FIXME Hmm - # temporary fallback - for algo in conflict_res: - new_res, curr_res = algo(new_branch, current_branch) - - if new_res < curr_res: - defer.returnValue(False) - elif new_res > curr_res: - defer.returnValue(True) - return + # We didn't find a common ancestor. This is probably fine. + pass - pdu_id = missing_prev.prev_state_id - origin = missing_prev.prev_state_origin + result = self._do_conflict_res(new_branch, current_branch) + defer.returnValue(result) - is_missing = yield self.store.get_pdu(pdu_id, origin) is None + def _do_conflict_res(self, new_branch, current_branch): + conflict_res = [ + self._do_power_level_conflict_res, + self._do_chain_length_conflict_res, + self._do_hash_conflict_res, + ] - if not is_missing: - raise Exception("Conflict resolution failed.") + for algo in conflict_res: + new_res, curr_res = algo(new_branch, current_branch) - yield self._replication.get_pdu( - destination=missing_prev.origin, - pdu_origin=origin, - pdu_id=pdu_id, - outlier=True - ) + if new_res < curr_res: + defer.returnValue(False) + elif new_res > curr_res: + defer.returnValue(True) - updated_current = yield self._handle_new_state(new_pdu) - defer.returnValue(updated_current) + raise Exception("Conflict resolution failed.") def _do_power_level_conflict_res(self, new_branch, current_branch): max_power_new = max( diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 0bf97e37ee..3cbce2d0a1 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -308,8 +308,8 @@ class PduStore(SQLBaseStore): @defer.inlineCallbacks def get_oldest_pdus_in_context(self, context): - """Get a list of Pdus that we haven't backfilled beyond yet (and haven't - seen). This list is used when we want to backfill backwards and is the + """Get a list of Pdus that we haven't backfilled beyond yet (and havent + seen). This list is used when we want to backfill backwards and is the list we send to the remote server. Args: @@ -524,13 +524,16 @@ class StatePduStore(SQLBaseStore): txn, new_pdu, current ) + missing_branch = None for branch, prev_state, state in enum_branches: if state: return_value[branch].append(state) else: + # We don't have prev_state :( + missing_branch = branch break - return return_value + return (return_value, missing_branch) def update_current_state(self, pdu_id, origin, context, pdu_type, state_key): diff --git a/tests/test_state.py b/tests/test_state.py index b01496c40f..4512475ebd 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -24,6 +24,8 @@ from collections import namedtuple from mock import Mock +import mock + ReturnType = namedtuple( "StateReturnType", ["new_branch", "current_branch"] @@ -54,7 +56,7 @@ class StateTestCase(unittest.TestCase): new_pdu = new_fake_pdu_entry("A", "test", "mem", "x", None, 10) self.persistence.get_unresolved_state_tree.return_value = ( - ReturnType([new_pdu], []) + (ReturnType([new_pdu], []), None) ) is_new = yield self.state.handle_new_state(new_pdu) @@ -78,7 +80,7 @@ class StateTestCase(unittest.TestCase): new_pdu = new_fake_pdu_entry("B", "test", "mem", "x", "A", 5) self.persistence.get_unresolved_state_tree.return_value = ( - ReturnType([new_pdu, old_pdu], [old_pdu]) + (ReturnType([new_pdu, old_pdu], [old_pdu]), None) ) is_new = yield self.state.handle_new_state(new_pdu) @@ -103,7 +105,7 @@ class StateTestCase(unittest.TestCase): new_pdu = new_fake_pdu_entry("C", "test", "mem", "x", "A", 5) self.persistence.get_unresolved_state_tree.return_value = ( - ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]) + (ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]), None) ) is_new = yield self.state.handle_new_state(new_pdu) @@ -128,7 +130,7 @@ class StateTestCase(unittest.TestCase): new_pdu = new_fake_pdu_entry("C", "test", "mem", "x", "A", 15) self.persistence.get_unresolved_state_tree.return_value = ( - ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]) + (ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]), None) ) is_new = yield self.state.handle_new_state(new_pdu) @@ -153,7 +155,7 @@ class StateTestCase(unittest.TestCase): new_pdu = new_fake_pdu_entry("C", "test", "mem", "x", "A", 10) self.persistence.get_unresolved_state_tree.return_value = ( - ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]) + (ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]), None) ) is_new = yield self.state.handle_new_state(new_pdu) @@ -179,7 +181,13 @@ class StateTestCase(unittest.TestCase): new_pdu = new_fake_pdu_entry("D", "test", "mem", "x", "C", 10) self.persistence.get_unresolved_state_tree.return_value = ( - ReturnType([new_pdu, old_pdu_3, old_pdu_1], [old_pdu_2, old_pdu_1]) + ( + ReturnType( + [new_pdu, old_pdu_3, old_pdu_1], + [old_pdu_2, old_pdu_1] + ), + None + ) ) is_new = yield self.state.handle_new_state(new_pdu) @@ -200,22 +208,32 @@ class StateTestCase(unittest.TestCase): # triggering a get_pdu request # The pdu we haven't seen - old_pdu_1 = new_fake_pdu_entry("A", "test", "mem", "x", None, 10) + old_pdu_1 = new_fake_pdu_entry( + "A", "test", "mem", "x", None, 10, depth=0 + ) - old_pdu_2 = new_fake_pdu_entry("B", "test", "mem", "x", None, 10) - new_pdu = new_fake_pdu_entry("C", "test", "mem", "x", "A", 20) + old_pdu_2 = new_fake_pdu_entry( + "B", "test", "mem", "x", "A", 10, depth=1 + ) + new_pdu = new_fake_pdu_entry( + "C", "test", "mem", "x", "A", 20, depth=2 + ) # The return_value of `get_unresolved_state_tree`, which changes after # the call to get_pdu - tree_to_return = [ReturnType([new_pdu], [old_pdu_2])] + tree_to_return = [(ReturnType([new_pdu], [old_pdu_2]), 0)] def return_tree(p): return tree_to_return[0] - def set_return_tree(*args, **kwargs): - tree_to_return[0] = ReturnType( - [new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1] + def set_return_tree(destination, pdu_origin, pdu_id, outlier=False): + tree_to_return[0] = ( + ReturnType( + [new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1] + ), + None ) + return defer.succeed(None) self.persistence.get_unresolved_state_tree.side_effect = return_tree @@ -227,6 +245,13 @@ class StateTestCase(unittest.TestCase): self.assertTrue(is_new) + self.replication.get_pdu.assert_called_with( + destination=new_pdu.origin, + pdu_origin=old_pdu_1.origin, + pdu_id=old_pdu_1.pdu_id, + outlier=True + ) + self.persistence.get_unresolved_state_tree.assert_called_with( new_pdu ) @@ -237,6 +262,184 @@ class StateTestCase(unittest.TestCase): self.assertEqual(1, self.persistence.update_current_state.call_count) + @defer.inlineCallbacks + def test_missing_pdu_depth_1(self): + # We try to update state against a PDU we haven't yet seen, + # triggering a get_pdu request + + # The pdu we haven't seen + old_pdu_1 = new_fake_pdu_entry( + "A", "test", "mem", "x", None, 10, depth=0 + ) + + old_pdu_2 = new_fake_pdu_entry( + "B", "test", "mem", "x", "A", 10, depth=2 + ) + old_pdu_3 = new_fake_pdu_entry( + "C", "test", "mem", "x", "B", 10, depth=3 + ) + new_pdu = new_fake_pdu_entry( + "D", "test", "mem", "x", "A", 20, depth=4 + ) + + # The return_value of `get_unresolved_state_tree`, which changes after + # the call to get_pdu + tree_to_return = [ + ( + ReturnType([new_pdu], [old_pdu_3]), + 0 + ), + ( + ReturnType( + [new_pdu, old_pdu_1], [old_pdu_3] + ), + 1 + ), + ( + ReturnType( + [new_pdu, old_pdu_1], [old_pdu_3, old_pdu_2, old_pdu_1] + ), + None + ), + ] + + to_return = [0] + + def return_tree(p): + return tree_to_return[to_return[0]] + + def set_return_tree(destination, pdu_origin, pdu_id, outlier=False): + to_return[0] += 1 + return defer.succeed(None) + + self.persistence.get_unresolved_state_tree.side_effect = return_tree + + self.replication.get_pdu.side_effect = set_return_tree + + self.persistence.get_pdu.return_value = None + + is_new = yield self.state.handle_new_state(new_pdu) + + self.assertTrue(is_new) + + self.assertEqual(2, self.replication.get_pdu.call_count) + + self.replication.get_pdu.assert_has_calls( + [ + mock.call( + destination=new_pdu.origin, + pdu_origin=old_pdu_1.origin, + pdu_id=old_pdu_1.pdu_id, + outlier=True + ), + mock.call( + destination=old_pdu_3.origin, + pdu_origin=old_pdu_2.origin, + pdu_id=old_pdu_2.pdu_id, + outlier=True + ), + ] + ) + + self.persistence.get_unresolved_state_tree.assert_called_with( + new_pdu + ) + + self.assertEquals( + 3, self.persistence.get_unresolved_state_tree.call_count + ) + + self.assertEqual(1, self.persistence.update_current_state.call_count) + + @defer.inlineCallbacks + def test_missing_pdu_depth_2(self): + # We try to update state against a PDU we haven't yet seen, + # triggering a get_pdu request + + # The pdu we haven't seen + old_pdu_1 = new_fake_pdu_entry( + "A", "test", "mem", "x", None, 10, depth=0 + ) + + old_pdu_2 = new_fake_pdu_entry( + "B", "test", "mem", "x", "A", 10, depth=2 + ) + old_pdu_3 = new_fake_pdu_entry( + "C", "test", "mem", "x", "B", 10, depth=3 + ) + new_pdu = new_fake_pdu_entry( + "D", "test", "mem", "x", "A", 20, depth=1 + ) + + # The return_value of `get_unresolved_state_tree`, which changes after + # the call to get_pdu + tree_to_return = [ + ( + ReturnType([new_pdu], [old_pdu_3]), + 1, + ), + ( + ReturnType( + [new_pdu], [old_pdu_3, old_pdu_2] + ), + 0, + ), + ( + ReturnType( + [new_pdu, old_pdu_1], [old_pdu_3, old_pdu_2, old_pdu_1] + ), + None + ), + ] + + to_return = [0] + + def return_tree(p): + return tree_to_return[to_return[0]] + + def set_return_tree(destination, pdu_origin, pdu_id, outlier=False): + to_return[0] += 1 + return defer.succeed(None) + + self.persistence.get_unresolved_state_tree.side_effect = return_tree + + self.replication.get_pdu.side_effect = set_return_tree + + self.persistence.get_pdu.return_value = None + + is_new = yield self.state.handle_new_state(new_pdu) + + self.assertTrue(is_new) + + self.assertEqual(2, self.replication.get_pdu.call_count) + + self.replication.get_pdu.assert_has_calls( + [ + mock.call( + destination=old_pdu_3.origin, + pdu_origin=old_pdu_2.origin, + pdu_id=old_pdu_2.pdu_id, + outlier=True + ), + mock.call( + destination=new_pdu.origin, + pdu_origin=old_pdu_1.origin, + pdu_id=old_pdu_1.pdu_id, + outlier=True + ), + ] + ) + + self.persistence.get_unresolved_state_tree.assert_called_with( + new_pdu + ) + + self.assertEquals( + 3, self.persistence.get_unresolved_state_tree.call_count + ) + + self.assertEqual(1, self.persistence.update_current_state.call_count) + @defer.inlineCallbacks def test_new_event(self): event = Mock() @@ -270,7 +473,7 @@ class StateTestCase(unittest.TestCase): def new_fake_pdu_entry(pdu_id, context, pdu_type, state_key, prev_state_id, - power_level): + power_level, depth=0): new_pdu = PduEntry( pdu_id=pdu_id, pdu_type=pdu_type, @@ -280,7 +483,7 @@ def new_fake_pdu_entry(pdu_id, context, pdu_type, state_key, prev_state_id, origin="example.com", context="context", ts=1405353060021, - depth=0, + depth=depth, content_json="{}", unrecognized_keys="{}", outlier=True, -- cgit 1.4.1 From 942d8412c49a1d481f0bedd189eb1598629b103c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Sep 2014 20:13:27 +0100 Subject: Handle the case where we don't have a common ancestor --- synapse/state.py | 27 ++++++++++++++++++--------- tests/test_state.py | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index e69282860a..0cc1344d51 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -174,7 +174,9 @@ class StateHandler(object): n = new_branch[-1] c = current_branch[-1] - if n.pdu_id == c.pdu_id and n.origin == c.origin: + common_ancestor = n.pdu_id == c.pdu_id and n.origin == c.origin + + if common_ancestor: # We found a common ancestor! if len(current_branch) == 1: @@ -185,10 +187,12 @@ class StateHandler(object): # We didn't find a common ancestor. This is probably fine. pass - result = self._do_conflict_res(new_branch, current_branch) + result = self._do_conflict_res( + new_branch, current_branch, common_ancestor + ) defer.returnValue(result) - def _do_conflict_res(self, new_branch, current_branch): + def _do_conflict_res(self, new_branch, current_branch, common_ancestor): conflict_res = [ self._do_power_level_conflict_res, self._do_chain_length_conflict_res, @@ -196,7 +200,9 @@ class StateHandler(object): ] for algo in conflict_res: - new_res, curr_res = algo(new_branch, current_branch) + new_res, curr_res = algo( + new_branch, current_branch, common_ancestor + ) if new_res < curr_res: defer.returnValue(False) @@ -205,23 +211,26 @@ class StateHandler(object): raise Exception("Conflict resolution failed.") - def _do_power_level_conflict_res(self, new_branch, current_branch): + def _do_power_level_conflict_res(self, new_branch, current_branch, + common_ancestor): max_power_new = max( - new_branch[:-1], + new_branch[:-1] if common_ancestor else new_branch, key=lambda t: t.power_level ).power_level max_power_current = max( - current_branch[:-1], + current_branch[:-1] if common_ancestor else current_branch, key=lambda t: t.power_level ).power_level return (max_power_new, max_power_current) - def _do_chain_length_conflict_res(self, new_branch, current_branch): + def _do_chain_length_conflict_res(self, new_branch, current_branch, + common_ancestor): return (len(new_branch), len(current_branch)) - def _do_hash_conflict_res(self, new_branch, current_branch): + def _do_hash_conflict_res(self, new_branch, current_branch, + common_ancestor): new_str = "".join([p.pdu_id + p.origin for p in new_branch]) c_str = "".join([p.pdu_id + p.origin for p in current_branch]) diff --git a/tests/test_state.py b/tests/test_state.py index 4512475ebd..a9fc3fb85c 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -440,6 +440,30 @@ class StateTestCase(unittest.TestCase): self.assertEqual(1, self.persistence.update_current_state.call_count) + @defer.inlineCallbacks + def test_no_common_ancestor(self): + # We do a direct overwriting of the old state, i.e., the new state + # points to the old state. + + old_pdu = new_fake_pdu_entry("A", "test", "mem", "x", None, 5) + new_pdu = new_fake_pdu_entry("B", "test", "mem", "x", None, 10) + + self.persistence.get_unresolved_state_tree.return_value = ( + (ReturnType([new_pdu], [old_pdu]), None) + ) + + is_new = yield self.state.handle_new_state(new_pdu) + + self.assertTrue(is_new) + + self.persistence.get_unresolved_state_tree.assert_called_once_with( + new_pdu + ) + + self.assertEqual(1, self.persistence.update_current_state.call_count) + + self.assertFalse(self.replication.get_pdu.called) + @defer.inlineCallbacks def test_new_event(self): event = Mock() -- cgit 1.4.1 From 76fe7d4eba334cee8b5c18ac26da709106dff1a2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 8 Sep 2014 12:11:36 -0700 Subject: Added num_joined_users key to /publicRooms for each room. Show this information in the webclient. --- synapse/handlers/room.py | 6 ++++++ webclient/app.css | 4 ++++ webclient/home/home.html | 5 ++++- webclient/recents/recents.html | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a0d0f2af16..310cb46fe7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -593,6 +593,12 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def get_public_room_list(self): chunk = yield self.store.get_rooms(is_public=True) + for room in chunk: + joined_members = yield self.store.get_room_members( + room_id=room["room_id"], + membership=Membership.JOIN + ) + room["num_joined_members"] = len(joined_members) # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) diff --git a/webclient/app.css b/webclient/app.css index 0c6ae9b668..b438cf0405 100755 --- a/webclient/app.css +++ b/webclient/app.css @@ -273,6 +273,10 @@ a:active { color: #000; } font-weight: bold; } +.publicRoomEntry { + margin-bottom: 5px; +} + /*** Participant list ***/ #usersTableWrapper { diff --git a/webclient/home/home.html b/webclient/home/home.html index 12b3c7f14e..cf6771814c 100644 --- a/webclient/home/home.html +++ b/webclient/home/home.html @@ -25,11 +25,14 @@

Public rooms

-
+
{{ room.room_display_name }} +
+ {{ room.num_joined_members }} {{ room.num_joined_members == 1 ? 'user' : 'users' }} +

diff --git a/webclient/recents/recents.html b/webclient/recents/recents.html index b903412815..efc5c39689 100644 --- a/webclient/recents/recents.html +++ b/webclient/recents/recents.html @@ -9,7 +9,7 @@ {{ room.room_id | mRoomName }} - {{ room.numUsersInRoom }} users + {{ room.numUsersInRoom }} {{ room.numUsersInRoom == 1 ? 'user' : 'users' }} {{ (room.lastMsg.ts) | date:'MMM d HH:mm' }} -- cgit 1.4.1 From e062f2dfa89eca20d409642b61bb240accb51bf1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Sep 2014 22:36:51 +0100 Subject: Apparently we can't do txn.rollback(), so raise and catch an exception instead. --- synapse/storage/__init__.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 8ed80109a5..a2eec3b209 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -47,6 +47,11 @@ import os logger = logging.getLogger(__name__) +class _RollbackButIsFineException(Exception): + """ This exception is used to rollback a transaction without implying + something went wrong. + """ + pass class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, @@ -71,13 +76,16 @@ class DataStore(RoomMemberStore, RoomStore, self.min_token -= 1 stream_ordering = self.min_token - latest = yield self._db_pool.runInteraction( - self._persist_pdu_event_txn, - pdu=pdu, - event=event, - backfilled=backfilled, - stream_ordering=stream_ordering, - ) + try: + latest = yield self._db_pool.runInteraction( + self._persist_pdu_event_txn, + pdu=pdu, + event=event, + backfilled=backfilled, + stream_ordering=stream_ordering, + ) + except _RollbackButIsFineException as e: + pass defer.returnValue(latest) @defer.inlineCallbacks @@ -175,12 +183,12 @@ class DataStore(RoomMemberStore, RoomStore, try: self._simple_insert_txn(txn, "events", vals) except: - logger.exception( + logger.warn( "Failed to persist, probably duplicate: %s", - event.event_id + event.event_id, + exc_info=True, ) - txn.rollback() - return + raise _RollbackButIsFineException("_persist_event") if not backfilled and hasattr(event, "state_key"): vals = { -- cgit 1.4.1 From a75f8686ba4c536db1a9e341786ac34bab3d25c7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Sep 2014 16:27:59 +0100 Subject: Fix bug where we used an unbound local variable if we ended up rolling back the persist_event transaction --- synapse/storage/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a2eec3b209..ad2a484c16 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -77,7 +77,7 @@ class DataStore(RoomMemberStore, RoomStore, stream_ordering = self.min_token try: - latest = yield self._db_pool.runInteraction( + yield self._db_pool.runInteraction( self._persist_pdu_event_txn, pdu=pdu, event=event, @@ -86,7 +86,6 @@ class DataStore(RoomMemberStore, RoomStore, ) except _RollbackButIsFineException as e: pass - defer.returnValue(latest) @defer.inlineCallbacks def get_event(self, event_id, allow_none=False): @@ -214,8 +213,6 @@ class DataStore(RoomMemberStore, RoomStore, } ) - return self._get_room_events_max_id_txn(txn) - @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): sql = ( -- cgit 1.4.1 From ce55a8cc4bb26c4518875743a04a06e792ad7ebf Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Sep 2014 15:42:15 +0100 Subject: Move database preparing code out of homserver.py into storage where it belongs --- synapse/app/homeserver.py | 73 ++++++--------------------------------------- synapse/server.py | 1 + synapse/storage/__init__.py | 61 +++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 64 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index d675d8c8f9..b63ecd4b5f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage import read_schema +from synapse.storage import prepare_database from synapse.server import HomeServer @@ -36,7 +36,6 @@ from daemonize import Daemonize import twisted.manhole.telnet import logging -import sqlite3 import os import re import sys @@ -44,22 +43,6 @@ import sys logger = logging.getLogger(__name__) -SCHEMAS = [ - "transactions", - "pdu", - "users", - "profiles", - "presence", - "im", - "room_aliases", -] - - -# Remember to update this number every time an incompatible change is made to -# database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 3 - - class SynapseHomeServer(HomeServer): def build_http_client(self): @@ -80,52 +63,12 @@ class SynapseHomeServer(HomeServer): ) def build_db_pool(self): - """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we - don't have to worry about overwriting existing content. - """ - logging.info("Preparing database: %s...", self.db_name) - - with sqlite3.connect(self.db_name) as db_conn: - c = db_conn.cursor() - c.execute("PRAGMA user_version") - row = c.fetchone() - - if row and row[0]: - user_version = row[0] - - if user_version > SCHEMA_VERSION: - raise ValueError("Cannot use this database as it is too " + - "new for the server to understand" - ) - elif user_version < SCHEMA_VERSION: - logging.info("Upgrading database from version %d", - user_version - ) - - # Run every version since after the current version. - for v in range(user_version + 1, SCHEMA_VERSION + 1): - sql_script = read_schema("delta/v%d" % (v)) - c.executescript(sql_script) - - db_conn.commit() - - else: - for sql_loc in SCHEMAS: - sql_script = read_schema(sql_loc) - - c.executescript(sql_script) - db_conn.commit() - c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION) - - c.close() - - logging.info("Database prepared in %s.", self.db_name) - - pool = adbapi.ConnectionPool( - 'sqlite3', self.db_name, check_same_thread=False, - cp_min=1, cp_max=1) - - return pool + return adbapi.ConnectionPool( + "sqlite3", self.get_db_name(), + check_same_thread=False, + cp_min=1, + cp_max=1 + ) def create_resource_tree(self, web_client, redirect_root_to_web_client): """Create the resource tree for this Home Server. @@ -270,6 +213,8 @@ def setup(): ) hs.start_listening(config.bind_port, config.unsecure_port) + prepare_database(hs.get_db_name()) + hs.get_db_pool() if config.manhole: diff --git a/synapse/server.py b/synapse/server.py index 83368ea5a7..1ba13f3df2 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -57,6 +57,7 @@ class BaseHomeServer(object): DEPENDENCIES = [ 'clock', 'http_client', + 'db_name', 'db_pool', 'persistence_service', 'replication_layer', diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index ad2a484c16..2543fb12b7 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -43,10 +43,28 @@ from .keys import KeyStore import json import logging import os +import sqlite3 logger = logging.getLogger(__name__) + +SCHEMAS = [ + "transactions", + "pdu", + "users", + "profiles", + "presence", + "im", + "room_aliases", +] + + +# Remember to update this number every time an incompatible change is made to +# database schema files, so the users will be informed on server restarts. +SCHEMA_VERSION = 3 + + class _RollbackButIsFineException(Exception): """ This exception is used to rollback a transaction without implying something went wrong. @@ -350,3 +368,46 @@ def read_schema(schema): """ with open(schema_path(schema)) as schema_file: return schema_file.read() + + +def prepare_database(db_name): + """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we + don't have to worry about overwriting existing content. + """ + logging.info("Preparing database: %s...", db_name) + + with sqlite3.connect(db_name) as db_conn: + c = db_conn.cursor() + c.execute("PRAGMA user_version") + row = c.fetchone() + + if row and row[0]: + user_version = row[0] + + if user_version > SCHEMA_VERSION: + raise ValueError("Cannot use this database as it is too " + + "new for the server to understand" + ) + elif user_version < SCHEMA_VERSION: + logging.info("Upgrading database from version %d", + user_version + ) + + # Run every version since after the current version. + for v in range(user_version + 1, SCHEMA_VERSION + 1): + sql_script = read_schema("delta/v%d" % (v)) + c.executescript(sql_script) + + db_conn.commit() + + else: + for sql_loc in SCHEMAS: + sql_script = read_schema(sql_loc) + + c.executescript(sql_script) + db_conn.commit() + c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION) + + c.close() + + logging.info("Database prepared in %s.", db_name) -- cgit 1.4.1 From 6c1f0055dc18038deb133ffad7718705e298c146 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Sep 2014 16:07:44 +0100 Subject: No need for a tiny run() function any more, just use reactor.run() directly --- synapse/app/homeserver.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b63ecd4b5f..e9a6321020 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -173,10 +173,6 @@ class SynapseHomeServer(HomeServer): logger.info("Synapse now listening on port %d", unsecure_port) -def run(): - reactor.run() - - def setup(): config = HomeServerConfig.load_config( "Synapse Homeserver", @@ -229,7 +225,7 @@ def setup(): daemon = Daemonize( app="synapse-homeserver", pid=config.pid_file, - action=run, + action=reactor.run, auto_close_fds=False, verbose=True, logger=logger, @@ -237,7 +233,7 @@ def setup(): daemon.start() else: - run() + reactor.run() if __name__ == '__main__': -- cgit 1.4.1 From 2faffc52eed70df7cf1adc021633f4a427917c90 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Sep 2014 16:16:24 +0100 Subject: Make sure not to open our TCP ports until /after/ the DB is nicely prepared ready for use --- synapse/app/homeserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index e9a6321020..e6377e3060 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -207,7 +207,6 @@ def setup(): web_client=config.webclient, redirect_root_to_web_client=True, ) - hs.start_listening(config.bind_port, config.unsecure_port) prepare_database(hs.get_db_name()) @@ -220,6 +219,8 @@ def setup(): f.namespace['hs'] = hs reactor.listenTCP(config.manhole, f, interface='127.0.0.1') + hs.start_listening(config.bind_port, config.unsecure_port) + if config.daemonize: print config.pid_file daemon = Daemonize( -- cgit 1.4.1 From 55397f634770f2b91cd4567e6b40507944144b67 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Sep 2014 16:23:58 +0100 Subject: prepare_database() on db_conn, not plain name, so we can pass in the connection from outside --- synapse/app/homeserver.py | 10 +++++++- synapse/storage/__init__.py | 57 +++++++++++++++++++++------------------------ 2 files changed, 35 insertions(+), 32 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index e6377e3060..2f1b954902 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -39,6 +39,7 @@ import logging import os import re import sys +import sqlite3 logger = logging.getLogger(__name__) @@ -208,7 +209,14 @@ def setup(): redirect_root_to_web_client=True, ) - prepare_database(hs.get_db_name()) + db_name = hs.get_db_name() + + logging.info("Preparing database: %s...", db_name) + + with sqlite3.connect(db_name) as db_conn: + prepare_database(db_conn) + + logging.info("Database prepared in %s.", db_name) hs.get_db_pool() diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2543fb12b7..6b273a0306 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -43,7 +43,6 @@ from .keys import KeyStore import json import logging import os -import sqlite3 logger = logging.getLogger(__name__) @@ -370,44 +369,40 @@ def read_schema(schema): return schema_file.read() -def prepare_database(db_name): +def prepare_database(db_conn): """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we don't have to worry about overwriting existing content. """ - logging.info("Preparing database: %s...", db_name) + c = db_conn.cursor() + c.execute("PRAGMA user_version") + row = c.fetchone() - with sqlite3.connect(db_name) as db_conn: - c = db_conn.cursor() - c.execute("PRAGMA user_version") - row = c.fetchone() + if row and row[0]: + user_version = row[0] - if row and row[0]: - user_version = row[0] - - if user_version > SCHEMA_VERSION: - raise ValueError("Cannot use this database as it is too " + - "new for the server to understand" - ) - elif user_version < SCHEMA_VERSION: - logging.info("Upgrading database from version %d", - user_version - ) + if user_version > SCHEMA_VERSION: + raise ValueError("Cannot use this database as it is too " + + "new for the server to understand" + ) + elif user_version < SCHEMA_VERSION: + logging.info("Upgrading database from version %d", + user_version + ) - # Run every version since after the current version. - for v in range(user_version + 1, SCHEMA_VERSION + 1): - sql_script = read_schema("delta/v%d" % (v)) - c.executescript(sql_script) + # Run every version since after the current version. + for v in range(user_version + 1, SCHEMA_VERSION + 1): + sql_script = read_schema("delta/v%d" % (v)) + c.executescript(sql_script) - db_conn.commit() + db_conn.commit() - else: - for sql_loc in SCHEMAS: - sql_script = read_schema(sql_loc) + else: + for sql_loc in SCHEMAS: + sql_script = read_schema(sql_loc) - c.executescript(sql_script) - db_conn.commit() - c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION) + c.executescript(sql_script) + db_conn.commit() + c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION) - c.close() + c.close() - logging.info("Database prepared in %s.", db_name) -- cgit 1.4.1 From aaf9ab68c6969d69150b7aa1f6ebddcf1c496050 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 11 Sep 2014 18:44:04 +0100 Subject: Rename _store_room_member_txn to _store_room_member_from_event_txn so we can create another, more sensible function of that name --- synapse/storage/__init__.py | 2 +- synapse/storage/roommember.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6b273a0306..8228069271 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -154,7 +154,7 @@ class DataStore(RoomMemberStore, RoomStore, @log_function def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None): if event.type == RoomMemberEvent.TYPE: - self._store_room_member_txn(txn, event) + self._store_room_member_from_event_txn(txn, event) elif event.type == FeedbackEvent.TYPE: self._store_feedback_txn(txn, event) elif event.type == RoomNameEvent.TYPE: diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 9a393e2568..437ff03a73 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class RoomMemberStore(SQLBaseStore): - def _store_room_member_txn(self, txn, event): + def _store_room_member_from_event_txn(self, txn, event): """Store a room member in the database. """ target_user_id = event.state_key -- cgit 1.4.1 From 249e8f227799c2b5f1adcd17a471ff9773b43f14 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 11 Sep 2014 18:52:35 +0100 Subject: Add a better _store_room_member_txn() method that takes separated fields instead of an event object; also add FIXME comment about a big bug in the logic --- synapse/storage/roommember.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 437ff03a73..b357dc3058 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -27,36 +27,49 @@ logger = logging.getLogger(__name__) class RoomMemberStore(SQLBaseStore): def _store_room_member_from_event_txn(self, txn, event): + self._store_room_member_txn(txn, + target_user_id=event.state_key, + sender_user_id=event.user_id, + room_id=event.room_id, + event_id=event.event_id, + membership=event.membership, + ) + + def _store_room_member_txn(self, txn, target_user_id, sender_user_id, + room_id, event_id, membership): """Store a room member in the database. """ - target_user_id = event.state_key domain = self.hs.parse_userid(target_user_id).domain self._simple_insert_txn( txn, "room_memberships", { - "event_id": event.event_id, + "event_id": event_id, "user_id": target_user_id, - "sender": event.user_id, - "room_id": event.room_id, - "membership": event.membership, + "sender": sender_user_id, + "room_id": room_id, + "membership": membership, } ) # Update room hosts table - if event.membership == Membership.JOIN: + # TODO(paul): This code is massively broken currently as it doesn't + # count users per room - meaning it'll delete on the FIRST user to + # have a membership other than JOIN - say, LEAVE, or even INVITE. + # FIXME + if membership == Membership.JOIN: sql = ( "INSERT OR IGNORE INTO room_hosts (room_id, host) " "VALUES (?, ?)" ) - txn.execute(sql, (event.room_id, domain)) + txn.execute(sql, (room_id, domain)) else: sql = ( "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" ) - txn.execute(sql, (event.room_id, domain)) + txn.execute(sql, (room_id, domain)) @defer.inlineCallbacks def get_room_member(self, user_id, room_id): -- cgit 1.4.1 From e53d77b5017e823506484bbb95964b4d97f3e2a1 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 12 Sep 2014 13:57:24 +0100 Subject: Add a .runInteraction() method on SQLBaseStore itself to wrap the .db_pool --- synapse/storage/__init__.py | 4 ++-- synapse/storage/_base.py | 20 ++++++++++++-------- synapse/storage/pdu.py | 24 ++++++++++++------------ synapse/storage/registration.py | 4 ++-- synapse/storage/room.py | 4 ++-- synapse/storage/roommember.py | 5 +++++ synapse/storage/stream.py | 2 +- synapse/storage/transactions.py | 12 ++++++------ 8 files changed, 42 insertions(+), 33 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 8228069271..629c110bed 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -94,7 +94,7 @@ class DataStore(RoomMemberStore, RoomStore, stream_ordering = self.min_token try: - yield self._db_pool.runInteraction( + yield self.runInteraction( self._persist_pdu_event_txn, pdu=pdu, event=event, @@ -297,7 +297,7 @@ class DataStore(RoomMemberStore, RoomStore, prev_state_pdu=prev_state_pdu, ) - return self._db_pool.runInteraction(_snapshot) + return self.runInteraction(_snapshot) class Snapshot(object): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 8037225079..8a36f0bc6a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -34,6 +34,10 @@ class SQLBaseStore(object): self.event_factory = hs.get_event_factory() self._clock = hs.get_clock() + def runInteraction(self, txn, *args, **kwargs): + """Wraps the .runInteraction() method on the underlying db_pool.""" + return self._db_pool.runInteraction(txn, *args, **kwargs) + def cursor_to_dict(self, cursor): """Converts a SQL cursor into an list of dicts. @@ -71,7 +75,7 @@ class SQLBaseStore(object): else: return cursor.fetchall() - return self._db_pool.runInteraction(interaction) + return self.runInteraction(interaction) def _execute_and_decode(self, query, *args): return self._execute(self.cursor_to_dict, query, *args) @@ -87,7 +91,7 @@ class SQLBaseStore(object): values : dict of new column names and values for them or_replace : bool; if True performs an INSERT OR REPLACE """ - return self._db_pool.runInteraction( + return self.runInteraction( self._simple_insert_txn, table, values, or_replace=or_replace ) @@ -164,7 +168,7 @@ class SQLBaseStore(object): txn.execute(sql, keyvalues.values()) return txn.fetchall() - res = yield self._db_pool.runInteraction(func) + res = yield self.runInteraction(func) defer.returnValue([r[0] for r in res]) @@ -187,7 +191,7 @@ class SQLBaseStore(object): txn.execute(sql, keyvalues.values()) return self.cursor_to_dict(txn) - return self._db_pool.runInteraction(func) + return self.runInteraction(func) def _simple_update_one(self, table, keyvalues, updatevalues, retcols=None): @@ -255,7 +259,7 @@ class SQLBaseStore(object): raise StoreError(500, "More than one row matched") return ret - return self._db_pool.runInteraction(func) + return self.runInteraction(func) def _simple_delete_one(self, table, keyvalues): """Executes a DELETE query on the named table, expecting to delete a @@ -276,7 +280,7 @@ class SQLBaseStore(object): raise StoreError(404, "No row found") if txn.rowcount > 1: raise StoreError(500, "more than one row matched") - return self._db_pool.runInteraction(func) + return self.runInteraction(func) def _simple_max_id(self, table): """Executes a SELECT query on the named table, expecting to return the @@ -294,7 +298,7 @@ class SQLBaseStore(object): return 0 return max_id - return self._db_pool.runInteraction(func) + return self.runInteraction(func) def _parse_event_from_row(self, row_dict): d = copy.deepcopy({k: v for k, v in row_dict.items() if v}) @@ -313,7 +317,7 @@ class SQLBaseStore(object): ) def _parse_events(self, rows): - return self._db_pool.runInteraction(self._parse_events_txn, rows) + return self.runInteraction(self._parse_events_txn, rows) def _parse_events_txn(self, txn, rows): events = [self._parse_event_from_row(r) for r in rows] diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 3cbce2d0a1..f770a82bcd 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -42,7 +42,7 @@ class PduStore(SQLBaseStore): PduTuple: If the pdu does not exist in the database, returns None """ - return self._db_pool.runInteraction( + return self.runInteraction( self._get_pdu_tuple, pdu_id, origin ) @@ -94,7 +94,7 @@ class PduStore(SQLBaseStore): list: A list of PduTuples """ - return self._db_pool.runInteraction( + return self.runInteraction( self._get_current_state_for_context, context ) @@ -142,7 +142,7 @@ class PduStore(SQLBaseStore): pdu_origin (str) """ - return self._db_pool.runInteraction( + return self.runInteraction( self._mark_as_processed, pdu_id, pdu_origin ) @@ -151,7 +151,7 @@ class PduStore(SQLBaseStore): def get_all_pdus_from_context(self, context): """Get a list of all PDUs for a given context.""" - return self._db_pool.runInteraction( + return self.runInteraction( self._get_all_pdus_from_context, context, ) @@ -178,7 +178,7 @@ class PduStore(SQLBaseStore): Return: list: A list of PduTuples """ - return self._db_pool.runInteraction( + return self.runInteraction( self._get_backfill, context, pdu_list, limit ) @@ -239,7 +239,7 @@ class PduStore(SQLBaseStore): txn context (str) """ - return self._db_pool.runInteraction( + return self.runInteraction( self._get_min_depth_for_context, context ) @@ -345,7 +345,7 @@ class PduStore(SQLBaseStore): bool """ - return self._db_pool.runInteraction( + return self.runInteraction( self._is_pdu_new, pdu_id=pdu_id, origin=origin, @@ -498,7 +498,7 @@ class StatePduStore(SQLBaseStore): ) def get_unresolved_state_tree(self, new_state_pdu): - return self._db_pool.runInteraction( + return self.runInteraction( self._get_unresolved_state_tree, new_state_pdu ) @@ -537,7 +537,7 @@ class StatePduStore(SQLBaseStore): def update_current_state(self, pdu_id, origin, context, pdu_type, state_key): - return self._db_pool.runInteraction( + return self.runInteraction( self._update_current_state, pdu_id, origin, context, pdu_type, state_key ) @@ -576,7 +576,7 @@ class StatePduStore(SQLBaseStore): PduEntry """ - return self._db_pool.runInteraction( + return self.runInteraction( self._get_current_state_pdu, context, pdu_type, state_key ) @@ -638,7 +638,7 @@ class StatePduStore(SQLBaseStore): PduIdTuple: A pdu that we are missing, or None if we have all the pdus required to do the conflict resolution. """ - return self._db_pool.runInteraction( + return self.runInteraction( self._get_next_missing_pdu, new_pdu ) @@ -682,7 +682,7 @@ class StatePduStore(SQLBaseStore): Returns: bool: True if the new_pdu clobbered the current state, False if not """ - return self._db_pool.runInteraction( + return self.runInteraction( self._handle_new_state, new_pdu ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index fd762bc643..db20b1daa0 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -62,7 +62,7 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if the user_id could not be registered. """ - yield self._db_pool.runInteraction(self._register, user_id, token, + yield self.runInteraction(self._register, user_id, token, password_hash) def _register(self, txn, user_id, token, password_hash): @@ -99,7 +99,7 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if no user was found. """ - user_id = yield self._db_pool.runInteraction(self._query_for_auth, + user_id = yield self.runInteraction(self._query_for_auth, token) defer.returnValue(user_id) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 017169ce00..5adf8cdf1b 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -149,7 +149,7 @@ class RoomStore(SQLBaseStore): defer.returnValue(None) def get_power_level(self, room_id, user_id): - return self._db_pool.runInteraction( + return self.runInteraction( self._get_power_level, room_id, user_id, ) @@ -182,7 +182,7 @@ class RoomStore(SQLBaseStore): return None def get_ops_levels(self, room_id): - return self._db_pool.runInteraction( + return self.runInteraction( self._get_ops_levels, room_id, ) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index b357dc3058..8cbc15356d 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -71,6 +71,11 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, (room_id, domain)) + def store_room_member(self, user_id, room_id, event_id, membership): + return self.runInteraction(self._store_room_member_txn, + user_id, user_id, room_id, event_id, membership + ) + @defer.inlineCallbacks def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index aff6dc9855..8c766b8a00 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -286,7 +286,7 @@ class StreamStore(SQLBaseStore): defer.returnValue(ret) def get_room_events_max_id(self): - return self._db_pool.runInteraction(self._get_room_events_max_id_txn) + return self.runInteraction(self._get_room_events_max_id_txn) def _get_room_events_max_id_txn(self, txn): txn.execute( diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 7467e1035b..ab4599b468 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -41,7 +41,7 @@ class TransactionStore(SQLBaseStore): this transaction or a 2-tuple of (int, dict) """ - return self._db_pool.runInteraction( + return self.runInteraction( self._get_received_txn_response, transaction_id, origin ) @@ -72,7 +72,7 @@ class TransactionStore(SQLBaseStore): response_json (str) """ - return self._db_pool.runInteraction( + return self.runInteraction( self._set_received_txn_response, transaction_id, origin, code, response_dict ) @@ -104,7 +104,7 @@ class TransactionStore(SQLBaseStore): list: A list of previous transaction ids. """ - return self._db_pool.runInteraction( + return self.runInteraction( self._prep_send_transaction, transaction_id, destination, ts, pdu_list ) @@ -159,7 +159,7 @@ class TransactionStore(SQLBaseStore): code (int) response_json (str) """ - return self._db_pool.runInteraction( + return self.runInteraction( self._delivered_txn, transaction_id, destination, code, response_dict ) @@ -184,7 +184,7 @@ class TransactionStore(SQLBaseStore): Returns: list: A list of `ReceivedTransactionsTable.EntryType` """ - return self._db_pool.runInteraction( + return self.runInteraction( self._get_transactions_after, transaction_id, destination ) @@ -214,7 +214,7 @@ class TransactionStore(SQLBaseStore): Returns list: A list of PduTuple """ - return self._db_pool.runInteraction( + return self.runInteraction( self._get_pdus_after_transaction, transaction_id, destination ) -- cgit 1.4.1 From 1c2024988457c5cdb9c0137a99e687e56e74e14b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 12 Sep 2014 14:37:55 +0100 Subject: Logging of all SQL queries via the 'synapse.storage.SQL' logger --- synapse/storage/_base.py | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 8a36f0bc6a..a46f2c6601 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -25,6 +25,44 @@ import json logger = logging.getLogger(__name__) +sql_logger = logging.getLogger("synapse.storage.SQL") + + +class LoggingTransaction(object): + """An object that almost-transparently proxies for the 'txn' object + passed to the constructor. Adds logging to the .execute() method.""" + __slots__ = ["txn"] + + def __init__(self, txn): + object.__setattr__(self, "txn", txn) + + def __getattribute__(self, name): + if name == "execute": + return object.__getattribute__(self, "execute") + + return getattr(object.__getattribute__(self, "txn"), name) + + def __setattr__(self, name, value): + setattr(object.__getattribute__(self, "txn"), name, value) + + def execute(self, sql, *args, **kwargs): + # TODO(paul): Maybe use 'info' and 'debug' for values? + sql_logger.debug("[SQL] %s", sql) + try: + if args and args[0]: + values = args[0] + sql_logger.debug("[SQL values] " + + ", ".join(("<%s>",) * len(values)), *values) + except: + # Don't let logging failures stop SQL from working + pass + + # TODO(paul): Here would be an excellent place to put some timing + # measurements, and log (warning?) slow queries. + return object.__getattribute__(self, "txn").execute( + sql, *args, **kwargs + ) + class SQLBaseStore(object): @@ -34,9 +72,12 @@ class SQLBaseStore(object): self.event_factory = hs.get_event_factory() self._clock = hs.get_clock() - def runInteraction(self, txn, *args, **kwargs): + def runInteraction(self, func, *args, **kwargs): """Wraps the .runInteraction() method on the underlying db_pool.""" - return self._db_pool.runInteraction(txn, *args, **kwargs) + def inner_func(txn, *args, **kwargs): + return func(LoggingTransaction(txn), *args, **kwargs) + + return self._db_pool.runInteraction(inner_func, *args, **kwargs) def cursor_to_dict(self, cursor): """Converts a SQL cursor into an list of dicts. -- cgit 1.4.1 From a840ff8f3fb0c58737a09cd326247fde4d75e2e3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 12 Sep 2014 14:38:27 +0100 Subject: Now don't need the other logger.debug() call in _execute --- synapse/storage/_base.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a46f2c6601..7006c1995b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -104,11 +104,6 @@ class SQLBaseStore(object): Returns: The result of decoder(results) """ - logger.debug( - "[SQL] %s Args=%s Func=%s", - query, args, decoder.__name__ if decoder else None - ) - def interaction(txn): cursor = txn.execute(query, args) if decoder: -- cgit 1.4.1 From a87eac4308e2230c2f79f41e2e1817636da4b208 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 12 Sep 2014 15:51:51 +0100 Subject: Revert recent changes to RoomMemberStore --- synapse/storage/__init__.py | 2 +- synapse/storage/roommember.py | 36 +++++++++--------------------------- 2 files changed, 10 insertions(+), 28 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 629c110bed..0dbae504b2 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -154,7 +154,7 @@ class DataStore(RoomMemberStore, RoomStore, @log_function def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None): if event.type == RoomMemberEvent.TYPE: - self._store_room_member_from_event_txn(txn, event) + self._store_room_member_txn(txn, event) elif event.type == FeedbackEvent.TYPE: self._store_feedback_txn(txn, event) elif event.type == RoomNameEvent.TYPE: diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 8cbc15356d..9a393e2568 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -26,55 +26,37 @@ logger = logging.getLogger(__name__) class RoomMemberStore(SQLBaseStore): - def _store_room_member_from_event_txn(self, txn, event): - self._store_room_member_txn(txn, - target_user_id=event.state_key, - sender_user_id=event.user_id, - room_id=event.room_id, - event_id=event.event_id, - membership=event.membership, - ) - - def _store_room_member_txn(self, txn, target_user_id, sender_user_id, - room_id, event_id, membership): + def _store_room_member_txn(self, txn, event): """Store a room member in the database. """ + target_user_id = event.state_key domain = self.hs.parse_userid(target_user_id).domain self._simple_insert_txn( txn, "room_memberships", { - "event_id": event_id, + "event_id": event.event_id, "user_id": target_user_id, - "sender": sender_user_id, - "room_id": room_id, - "membership": membership, + "sender": event.user_id, + "room_id": event.room_id, + "membership": event.membership, } ) # Update room hosts table - # TODO(paul): This code is massively broken currently as it doesn't - # count users per room - meaning it'll delete on the FIRST user to - # have a membership other than JOIN - say, LEAVE, or even INVITE. - # FIXME - if membership == Membership.JOIN: + if event.membership == Membership.JOIN: sql = ( "INSERT OR IGNORE INTO room_hosts (room_id, host) " "VALUES (?, ?)" ) - txn.execute(sql, (room_id, domain)) + txn.execute(sql, (event.room_id, domain)) else: sql = ( "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" ) - txn.execute(sql, (room_id, domain)) - - def store_room_member(self, user_id, room_id, event_id, membership): - return self.runInteraction(self._store_room_member_txn, - user_id, user_id, room_id, event_id, membership - ) + txn.execute(sql, (event.room_id, domain)) @defer.inlineCallbacks def get_room_member(self, user_id, room_id): -- cgit 1.4.1 From aa525e4a634a2a5932995dab3faead5a1b91c5a7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 12 Sep 2014 16:43:49 +0100 Subject: More accurate docs / clearer paramter names in RoomMemberStore --- synapse/storage/roommember.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 9a393e2568..8357a0edd7 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -120,7 +120,7 @@ class RoomMemberStore(SQLBaseStore): membership_list (list): A list of synapse.api.constants.Membership values which the user must be in. Returns: - A list of dicts with "room_id" and "membership" keys. + A list of RoomMemberEvent objects """ if not membership_list: return defer.succeed(None) @@ -165,10 +165,11 @@ class RoomMemberStore(SQLBaseStore): defer.returnValue(results) @defer.inlineCallbacks - def user_rooms_intersect(self, user_list): - """ Checks whether a list of users share a room. + def user_rooms_intersect(self, user_id_list): + """ Checks whether all the users whose IDs are given in a list share a + room. """ - user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_list)) + user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) sql = ( "SELECT m.room_id FROM room_memberships as m " "INNER JOIN current_state_events as c " @@ -178,8 +179,8 @@ class RoomMemberStore(SQLBaseStore): "GROUP BY m.room_id HAVING COUNT(m.room_id) = ?" ) % {"clause": user_list_clause} - args = user_list - args.append(len(user_list)) + args = list(user_id_list) + args.append(len(user_id_list)) rows = yield self._execute(None, sql, *args) -- cgit 1.4.1 From ca1ae7cf9b4c75f677f453be1fb7d9a06c17194d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Sep 2014 13:54:13 +0100 Subject: Fix bug where we didn't return a tuple when expected. --- synapse/storage/pdu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 3cbce2d0a1..f780111b3b 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -516,7 +516,7 @@ class StatePduStore(SQLBaseStore): if not current: logger.debug("get_unresolved_state_tree No current state.") - return return_value + return (return_value, None) return_value.current_branch.append(current) -- cgit 1.4.1 From b42fe05c516ffc8e049ab9b56451cceb813bdf64 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Sep 2014 17:09:55 +0100 Subject: Fix bug where we incorrectly removed a remote host from the list of hosts in a room when any user from that host left that room even if they weren't the last user from that host in that room --- synapse/storage/roommember.py | 57 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 12 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 9a393e2568..20f22057a2 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import SQLBaseStore from synapse.api.constants import Membership +from synapse.util.logutils import log_function import logging @@ -29,8 +30,18 @@ class RoomMemberStore(SQLBaseStore): def _store_room_member_txn(self, txn, event): """Store a room member in the database. """ - target_user_id = event.state_key - domain = self.hs.parse_userid(target_user_id).domain + try: + target_user_id = event.state_key + domain = self.hs.parse_userid(target_user_id).domain + except: + logger.exception("Failed to parse target_user_id=%s", target_user_id) + raise + + logger.debug( + "_store_room_member_txn: target_user_id=%s, membership=%s", + target_user_id, + event.membership, + ) self._simple_insert_txn( txn, @@ -51,12 +62,30 @@ class RoomMemberStore(SQLBaseStore): "VALUES (?, ?)" ) txn.execute(sql, (event.room_id, domain)) - else: - sql = ( - "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" + elif event.membership != Membership.INVITE: + # Check if this was the last person to have left. + member_events = self._get_members_query_txn( + txn, + where_clause="c.room_id = ? AND m.membership = ?", + where_values=(event.room_id, Membership.JOIN,) ) - txn.execute(sql, (event.room_id, domain)) + joined_domains = set() + for e in member_events: + try: + joined_domains.add( + self.hs.parse_userid(e.state_key).domain + ) + except: + # FIXME: How do we deal with invalid user ids in the db? + logger.exception("Invalid user_id: %s", event.state_key) + + if domain not in joined_domains: + sql = ( + "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" + ) + + txn.execute(sql, (event.room_id, domain)) @defer.inlineCallbacks def get_room_member(self, user_id, room_id): @@ -146,8 +175,13 @@ class RoomMemberStore(SQLBaseStore): vals = where_dict.values() return self._get_members_query(clause, vals) - @defer.inlineCallbacks def _get_members_query(self, where_clause, where_values): + return self._db_pool.runInteraction( + self._get_members_query_txn, + where_clause, where_values + ) + + def _get_members_query_txn(self, txn, where_clause, where_values): sql = ( "SELECT e.* FROM events as e " "INNER JOIN room_memberships as m " @@ -157,12 +191,11 @@ class RoomMemberStore(SQLBaseStore): "WHERE %s " ) % (where_clause,) - rows = yield self._execute_and_decode(sql, *where_values) - - # logger.debug("_get_members_query Got rows %s", rows) + txn.execute(sql, where_values) + rows = self.cursor_to_dict(txn) - results = yield self._parse_events(rows) - defer.returnValue(results) + results = self._parse_events_txn(txn, rows) + return results @defer.inlineCallbacks def user_rooms_intersect(self, user_list): -- cgit 1.4.1 From 39e3fc69e5a190371aa6936bfea57e9f8bd5255b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Sep 2014 17:11:00 +0100 Subject: Make the state resolution use actual power levels rather than taking them from a Pdu key. --- synapse/federation/units.py | 1 + synapse/state.py | 46 ++++++++--- synapse/storage/_base.py | 8 ++ synapse/storage/pdu.py | 81 +++---------------- tests/test_state.py | 185 +++++++++++++++++++++++++++++++++----------- 5 files changed, 194 insertions(+), 127 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 9740431279..622fe66a8f 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -69,6 +69,7 @@ class Pdu(JsonEncodedObject): "prev_state_id", "prev_state_origin", "required_power_level", + "user_id", ] internal_keys = [ diff --git a/synapse/state.py b/synapse/state.py index 0cc1344d51..9db84c9b5c 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -115,6 +115,8 @@ class StateHandler(object): is_new = yield self._handle_new_state(new_pdu) + logger.debug("is_new: %s %s %s", is_new, new_pdu.pdu_id, new_pdu.origin) + if is_new: yield self.store.update_current_state( pdu_id=new_pdu.pdu_id, @@ -187,11 +189,12 @@ class StateHandler(object): # We didn't find a common ancestor. This is probably fine. pass - result = self._do_conflict_res( + result = yield self._do_conflict_res( new_branch, current_branch, common_ancestor ) defer.returnValue(result) + @defer.inlineCallbacks def _do_conflict_res(self, new_branch, current_branch, common_ancestor): conflict_res = [ self._do_power_level_conflict_res, @@ -200,7 +203,8 @@ class StateHandler(object): ] for algo in conflict_res: - new_res, curr_res = algo( + new_res, curr_res = yield defer.maybeDeferred( + algo, new_branch, current_branch, common_ancestor ) @@ -211,19 +215,39 @@ class StateHandler(object): raise Exception("Conflict resolution failed.") + @defer.inlineCallbacks def _do_power_level_conflict_res(self, new_branch, current_branch, common_ancestor): - max_power_new = max( - new_branch[:-1] if common_ancestor else new_branch, - key=lambda t: t.power_level - ).power_level + new_powers_deferreds = [] + for e in new_branch[:-1] if common_ancestor else new_branch: + if hasattr(e, "user_id"): + new_powers_deferreds.append( + self.store.get_power_level(e.context, e.user_id) + ) + + current_powers_deferreds = [] + for e in current_branch[:-1] if common_ancestor else current_branch: + if hasattr(e, "user_id"): + current_powers_deferreds.append( + self.store.get_power_level(e.context, e.user_id) + ) + + new_powers = yield defer.gatherResults( + new_powers_deferreds, + consumeErrors=True + ) - max_power_current = max( - current_branch[:-1] if common_ancestor else current_branch, - key=lambda t: t.power_level - ).power_level + current_powers = yield defer.gatherResults( + current_powers_deferreds, + consumeErrors=True + ) + + max_power_new = max(new_powers) + max_power_current = max(current_powers) - return (max_power_new, max_power_current) + defer.returnValue( + (max_power_new, max_power_current) + ) def _do_chain_length_conflict_res(self, new_branch, current_branch, common_ancestor): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 8037225079..8deaaf93bd 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -17,6 +17,7 @@ import logging from twisted.internet import defer from synapse.api.errors import StoreError +from synapse.util.logutils import log_function import collections import copy @@ -91,6 +92,7 @@ class SQLBaseStore(object): self._simple_insert_txn, table, values, or_replace=or_replace ) + @log_function def _simple_insert_txn(self, txn, table, values, or_replace=False): sql = "%s INTO %s (%s) VALUES(%s)" % ( ("INSERT OR REPLACE" if or_replace else "INSERT"), @@ -98,6 +100,12 @@ class SQLBaseStore(object): ", ".join(k for k in values), ", ".join("?" for k in values) ) + + logger.debug( + "[SQL] %s Args=%s Func=%s", + sql, values.values(), + ) + txn.execute(sql, values.values()) return txn.lastrowid diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index f780111b3b..3c859fdeac 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -17,6 +17,7 @@ from twisted.internet import defer from ._base import SQLBaseStore, Table, JoinHelper +from synapse.federation.units import Pdu from synapse.util.logutils import log_function from collections import namedtuple @@ -625,53 +626,6 @@ class StatePduStore(SQLBaseStore): return result - def get_next_missing_pdu(self, new_pdu): - """When we get a new state pdu we need to check whether we need to do - any conflict resolution, if we do then we need to check if we need - to go back and request some more state pdus that we haven't seen yet. - - Args: - txn - new_pdu - - Returns: - PduIdTuple: A pdu that we are missing, or None if we have all the - pdus required to do the conflict resolution. - """ - return self._db_pool.runInteraction( - self._get_next_missing_pdu, new_pdu - ) - - def _get_next_missing_pdu(self, txn, new_pdu): - logger.debug( - "get_next_missing_pdu %s %s", - new_pdu.pdu_id, new_pdu.origin - ) - - current = self._get_current_interaction( - txn, - new_pdu.context, new_pdu.pdu_type, new_pdu.state_key - ) - - if (not current or not current.prev_state_id - or not current.prev_state_origin): - return None - - # Oh look, it's a straight clobber, so wooooo almost no-op. - if (new_pdu.prev_state_id == current.pdu_id - and new_pdu.prev_state_origin == current.origin): - return None - - enum_branches = self._enumerate_state_branches(txn, new_pdu, current) - for branch, prev_state, state in enum_branches: - if not state: - return PduIdTuple( - prev_state.prev_state_id, - prev_state.prev_state_origin - ) - - return None - def handle_new_state(self, new_pdu): """Actually perform conflict resolution on the new_pdu on the assumption we have all the pdus required to perform it. @@ -755,24 +709,11 @@ class StatePduStore(SQLBaseStore): return is_current - @classmethod @log_function - def _enumerate_state_branches(cls, txn, pdu_a, pdu_b): + def _enumerate_state_branches(self, txn, pdu_a, pdu_b): branch_a = pdu_a branch_b = pdu_b - get_query = ( - "SELECT %(fields)s FROM %(pdus)s as p " - "LEFT JOIN %(state)s as s " - "ON p.pdu_id = s.pdu_id AND p.origin = s.origin " - "WHERE p.pdu_id = ? AND p.origin = ? " - ) % { - "fields": _pdu_state_joiner.get_fields( - PdusTable="p", StatePdusTable="s"), - "pdus": PdusTable.table_name, - "state": StatePdusTable.table_name, - } - while True: if (branch_a.pdu_id == branch_b.pdu_id and branch_a.origin == branch_b.origin): @@ -804,13 +745,12 @@ class StatePduStore(SQLBaseStore): branch_a.prev_state_origin ) - logger.debug("getting branch_a prev %s", pdu_tuple) - txn.execute(get_query, pdu_tuple) - prev_branch = branch_a - res = txn.fetchone() - branch_a = PduEntry(*res) if res else None + logger.debug("getting branch_a prev %s", pdu_tuple) + branch_a = self._get_pdu_tuple(txn, *pdu_tuple) + if branch_a: + branch_a = Pdu.from_pdu_tuple(branch_a) logger.debug("branch_a=%s", branch_a) @@ -823,14 +763,13 @@ class StatePduStore(SQLBaseStore): branch_b.prev_state_id, branch_b.prev_state_origin ) - txn.execute(get_query, pdu_tuple) - - logger.debug("getting branch_b prev %s", pdu_tuple) prev_branch = branch_b - res = txn.fetchone() - branch_b = PduEntry(*res) if res else None + logger.debug("getting branch_b prev %s", pdu_tuple) + branch_b = self._get_pdu_tuple(txn, *pdu_tuple) + if branch_b: + branch_b = Pdu.from_pdu_tuple(branch_b) logger.debug("branch_b=%s", branch_b) diff --git a/tests/test_state.py b/tests/test_state.py index a9fc3fb85c..16af95b7bc 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -15,15 +15,18 @@ from twisted.internet import defer from twisted.trial import unittest +from twisted.python.log import PythonLoggingObserver from synapse.state import StateHandler from synapse.storage.pdu import PduEntry from synapse.federation.pdu_codec import encode_event_id +from synapse.federation.units import Pdu from collections import namedtuple from mock import Mock +import logging import mock @@ -32,6 +35,11 @@ ReturnType = namedtuple( ) +def _gen_get_power_level(power_level_list): + def get_power_level(room_id, user_id): + return defer.succeed(power_level_list.get(user_id, None)) + return get_power_level + class StateTestCase(unittest.TestCase): def setUp(self): self.persistence = Mock(spec=[ @@ -40,6 +48,7 @@ class StateTestCase(unittest.TestCase): "get_latest_pdus_in_context", "get_current_state_pdu", "get_pdu", + "get_power_level", ]) self.replication = Mock(spec=["get_pdu"]) @@ -53,7 +62,9 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_new_state_key(self): # We've never seen anything for this state before - new_pdu = new_fake_pdu_entry("A", "test", "mem", "x", None, 10) + new_pdu = new_fake_pdu("A", "test", "mem", "x", None, "u") + + self.persistence.get_power_level.side_effect = _gen_get_power_level({}) self.persistence.get_unresolved_state_tree.return_value = ( (ReturnType([new_pdu], []), None) @@ -76,8 +87,13 @@ class StateTestCase(unittest.TestCase): # We do a direct overwriting of the old state, i.e., the new state # points to the old state. - old_pdu = new_fake_pdu_entry("A", "test", "mem", "x", None, 10) - new_pdu = new_fake_pdu_entry("B", "test", "mem", "x", "A", 5) + old_pdu = new_fake_pdu("A", "test", "mem", "x", None, "u1") + new_pdu = new_fake_pdu("B", "test", "mem", "x", "A", "u2") + + self.persistence.get_power_level.side_effect = _gen_get_power_level({ + "u1": 10, + "u2": 5, + }) self.persistence.get_unresolved_state_tree.return_value = ( (ReturnType([new_pdu, old_pdu], [old_pdu]), None) @@ -95,14 +111,48 @@ class StateTestCase(unittest.TestCase): self.assertFalse(self.replication.get_pdu.called) + @defer.inlineCallbacks + def test_overwrite(self): + old_pdu_1 = new_fake_pdu("A", "test", "mem", "x", None, "u1") + old_pdu_2 = new_fake_pdu("B", "test", "mem", "x", "A", "u2") + new_pdu = new_fake_pdu("C", "test", "mem", "x", "B", "u3") + + self.persistence.get_power_level.side_effect = _gen_get_power_level({ + "u1": 10, + "u2": 5, + "u3": 0, + }) + + self.persistence.get_unresolved_state_tree.return_value = ( + (ReturnType([new_pdu, old_pdu_2, old_pdu_1], [old_pdu_1]), None) + ) + + is_new = yield self.state.handle_new_state(new_pdu) + + self.assertTrue(is_new) + + self.persistence.get_unresolved_state_tree.assert_called_once_with( + new_pdu + ) + + self.assertEqual(1, self.persistence.update_current_state.call_count) + + self.assertFalse(self.replication.get_pdu.called) + @defer.inlineCallbacks def test_power_level_fail(self): # We try to update the state based on an outdated state, and have a # too low power level. - old_pdu_1 = new_fake_pdu_entry("A", "test", "mem", "x", None, 10) - old_pdu_2 = new_fake_pdu_entry("B", "test", "mem", "x", None, 10) - new_pdu = new_fake_pdu_entry("C", "test", "mem", "x", "A", 5) + old_pdu_1 = new_fake_pdu("A", "test", "mem", "x", None, "u1") + old_pdu_2 = new_fake_pdu("B", "test", "mem", "x", None, "u2") + new_pdu = new_fake_pdu("C", "test", "mem", "x", "A", "u3") + + self.persistence.get_power_level.side_effect = _gen_get_power_level({ + "u1": 10, + "u2": 10, + "u3": 5, + }) self.persistence.get_unresolved_state_tree.return_value = ( (ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]), None) @@ -125,9 +175,15 @@ class StateTestCase(unittest.TestCase): # We try to update the state based on an outdated state, but have # sufficient power level to force the update. - old_pdu_1 = new_fake_pdu_entry("A", "test", "mem", "x", None, 10) - old_pdu_2 = new_fake_pdu_entry("B", "test", "mem", "x", None, 10) - new_pdu = new_fake_pdu_entry("C", "test", "mem", "x", "A", 15) + old_pdu_1 = new_fake_pdu("A", "test", "mem", "x", None, "u1") + old_pdu_2 = new_fake_pdu("B", "test", "mem", "x", None, "u2") + new_pdu = new_fake_pdu("C", "test", "mem", "x", "A", "u3") + + self.persistence.get_power_level.side_effect = _gen_get_power_level({ + "u1": 10, + "u2": 10, + "u3": 15, + }) self.persistence.get_unresolved_state_tree.return_value = ( (ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]), None) @@ -150,9 +206,15 @@ class StateTestCase(unittest.TestCase): # We try to update the state based on an outdated state, the power # levels are the same and so are the branch lengths - old_pdu_1 = new_fake_pdu_entry("A", "test", "mem", "x", None, 10) - old_pdu_2 = new_fake_pdu_entry("B", "test", "mem", "x", None, 10) - new_pdu = new_fake_pdu_entry("C", "test", "mem", "x", "A", 10) + old_pdu_1 = new_fake_pdu("A", "test", "mem", "x", None, "u1") + old_pdu_2 = new_fake_pdu("B", "test", "mem", "x", None, "u2") + new_pdu = new_fake_pdu("C", "test", "mem", "x", "A", "u3") + + self.persistence.get_power_level.side_effect = _gen_get_power_level({ + "u1": 10, + "u2": 10, + "u3": 10, + }) self.persistence.get_unresolved_state_tree.return_value = ( (ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]), None) @@ -175,10 +237,17 @@ class StateTestCase(unittest.TestCase): # We try to update the state based on an outdated state, the power # levels are the same but the branch length of the new one is longer. - old_pdu_1 = new_fake_pdu_entry("A", "test", "mem", "x", None, 10) - old_pdu_2 = new_fake_pdu_entry("B", "test", "mem", "x", None, 10) - old_pdu_3 = new_fake_pdu_entry("C", "test", "mem", "x", "A", 10) - new_pdu = new_fake_pdu_entry("D", "test", "mem", "x", "C", 10) + old_pdu_1 = new_fake_pdu("A", "test", "mem", "x", None, "u1") + old_pdu_2 = new_fake_pdu("B", "test", "mem", "x", None, "u2") + old_pdu_3 = new_fake_pdu("C", "test", "mem", "x", "A", "u3") + new_pdu = new_fake_pdu("D", "test", "mem", "x", "C", "u4") + + self.persistence.get_power_level.side_effect = _gen_get_power_level({ + "u1": 10, + "u2": 10, + "u3": 10, + "u4": 10, + }) self.persistence.get_unresolved_state_tree.return_value = ( ( @@ -208,17 +277,23 @@ class StateTestCase(unittest.TestCase): # triggering a get_pdu request # The pdu we haven't seen - old_pdu_1 = new_fake_pdu_entry( - "A", "test", "mem", "x", None, 10, depth=0 + old_pdu_1 = new_fake_pdu( + "A", "test", "mem", "x", None, "u1", depth=0 ) - old_pdu_2 = new_fake_pdu_entry( - "B", "test", "mem", "x", "A", 10, depth=1 + old_pdu_2 = new_fake_pdu( + "B", "test", "mem", "x", "A", "u2", depth=1 ) - new_pdu = new_fake_pdu_entry( - "C", "test", "mem", "x", "A", 20, depth=2 + new_pdu = new_fake_pdu( + "C", "test", "mem", "x", "A", "u3", depth=2 ) + self.persistence.get_power_level.side_effect = _gen_get_power_level({ + "u1": 10, + "u2": 10, + "u3": 20, + }) + # The return_value of `get_unresolved_state_tree`, which changes after # the call to get_pdu tree_to_return = [(ReturnType([new_pdu], [old_pdu_2]), 0)] @@ -268,20 +343,27 @@ class StateTestCase(unittest.TestCase): # triggering a get_pdu request # The pdu we haven't seen - old_pdu_1 = new_fake_pdu_entry( - "A", "test", "mem", "x", None, 10, depth=0 + old_pdu_1 = new_fake_pdu( + "A", "test", "mem", "x", None, "u1", depth=0 ) - old_pdu_2 = new_fake_pdu_entry( - "B", "test", "mem", "x", "A", 10, depth=2 + old_pdu_2 = new_fake_pdu( + "B", "test", "mem", "x", "A", "u2", depth=2 ) - old_pdu_3 = new_fake_pdu_entry( - "C", "test", "mem", "x", "B", 10, depth=3 + old_pdu_3 = new_fake_pdu( + "C", "test", "mem", "x", "B", "u3", depth=3 ) - new_pdu = new_fake_pdu_entry( - "D", "test", "mem", "x", "A", 20, depth=4 + new_pdu = new_fake_pdu( + "D", "test", "mem", "x", "A", "u4", depth=4 ) + self.persistence.get_power_level.side_effect = _gen_get_power_level({ + "u1": 10, + "u2": 10, + "u3": 10, + "u4": 20, + }) + # The return_value of `get_unresolved_state_tree`, which changes after # the call to get_pdu tree_to_return = [ @@ -357,20 +439,27 @@ class StateTestCase(unittest.TestCase): # triggering a get_pdu request # The pdu we haven't seen - old_pdu_1 = new_fake_pdu_entry( - "A", "test", "mem", "x", None, 10, depth=0 + old_pdu_1 = new_fake_pdu( + "A", "test", "mem", "x", None, "u1", depth=0 ) - old_pdu_2 = new_fake_pdu_entry( - "B", "test", "mem", "x", "A", 10, depth=2 + old_pdu_2 = new_fake_pdu( + "B", "test", "mem", "x", "A", "u2", depth=2 ) - old_pdu_3 = new_fake_pdu_entry( - "C", "test", "mem", "x", "B", 10, depth=3 + old_pdu_3 = new_fake_pdu( + "C", "test", "mem", "x", "B", "u3", depth=3 ) - new_pdu = new_fake_pdu_entry( - "D", "test", "mem", "x", "A", 20, depth=1 + new_pdu = new_fake_pdu( + "D", "test", "mem", "x", "A", "u4", depth=1 ) + self.persistence.get_power_level.side_effect = _gen_get_power_level({ + "u1": 10, + "u2": 10, + "u3": 10, + "u4": 20, + }) + # The return_value of `get_unresolved_state_tree`, which changes after # the call to get_pdu tree_to_return = [ @@ -445,8 +534,13 @@ class StateTestCase(unittest.TestCase): # We do a direct overwriting of the old state, i.e., the new state # points to the old state. - old_pdu = new_fake_pdu_entry("A", "test", "mem", "x", None, 5) - new_pdu = new_fake_pdu_entry("B", "test", "mem", "x", None, 10) + old_pdu = new_fake_pdu("A", "test", "mem", "x", None, "u1") + new_pdu = new_fake_pdu("B", "test", "mem", "x", None, "u2") + + self.persistence.get_power_level.side_effect = _gen_get_power_level({ + "u1": 5, + "u2": 10, + }) self.persistence.get_unresolved_state_tree.return_value = ( (ReturnType([new_pdu], [old_pdu]), None) @@ -469,7 +563,7 @@ class StateTestCase(unittest.TestCase): event = Mock() event.event_id = "12123123@test" - state_pdu = new_fake_pdu_entry("C", "test", "mem", "x", "A", 20) + state_pdu = new_fake_pdu("C", "test", "mem", "x", "A", 20) snapshot = Mock() snapshot.prev_state_pdu = state_pdu @@ -496,13 +590,13 @@ class StateTestCase(unittest.TestCase): ) -def new_fake_pdu_entry(pdu_id, context, pdu_type, state_key, prev_state_id, - power_level, depth=0): - new_pdu = PduEntry( +def new_fake_pdu(pdu_id, context, pdu_type, state_key, prev_state_id, + user_id, depth=0): + new_pdu = Pdu( pdu_id=pdu_id, pdu_type=pdu_type, state_key=state_key, - power_level=power_level, + user_id=user_id, prev_state_id=prev_state_id, origin="example.com", context="context", @@ -514,6 +608,7 @@ def new_fake_pdu_entry(pdu_id, context, pdu_type, state_key, prev_state_id, is_state=True, prev_state_origin="example.com", have_processed=True, + content={}, ) return new_pdu -- cgit 1.4.1 From 667e747ed11a418da317a03fc3c59a205c5c4af0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Sep 2014 17:56:21 +0100 Subject: Fix bug where we no longer stored user_id on Pdus --- synapse/storage/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index ad2a484c16..9201a377b6 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -36,7 +36,7 @@ from .registration import RegistrationStore from .room import RoomStore from .roommember import RoomMemberStore from .stream import StreamStore -from .pdu import StatePduStore, PduStore +from .pdu import StatePduStore, PduStore, PdusTable from .transactions import TransactionStore from .keys import KeyStore @@ -123,6 +123,12 @@ class DataStore(RoomMemberStore, RoomStore, del cols["content"] del cols["prev_pdus"] cols["content_json"] = json.dumps(pdu.content) + + unrec_keys.update({ + k: v for k, v in cols.items() + if k not in PdusTable.fields + }) + cols["unrecognized_keys"] = json.dumps(unrec_keys) logger.debug("Persisting: %s", repr(cols)) -- cgit 1.4.1 From 14975ce5bcf4dac2720cc4be290100a580334393 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Sep 2014 17:57:02 +0100 Subject: Fix bug where we relied on the current_state_events being updated when we are handling type specific persistence --- synapse/storage/roommember.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 20f22057a2..676b2f2653 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -66,8 +66,8 @@ class RoomMemberStore(SQLBaseStore): # Check if this was the last person to have left. member_events = self._get_members_query_txn( txn, - where_clause="c.room_id = ? AND m.membership = ?", - where_values=(event.room_id, Membership.JOIN,) + where_clause="c.room_id = ? AND m.membership = ? AND m.user_id != ?", + where_values=(event.room_id, Membership.JOIN, target_user_id,) ) joined_domains = set() -- cgit 1.4.1 From afb7f173cf3f04ffe212025910a950814d1761bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Sep 2014 18:13:05 +0100 Subject: Bump version and change log --- CHANGES.rst | 19 +++++++++++++++++++ VERSION | 2 +- synapse/__init__.py | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/CHANGES.rst b/CHANGES.rst index b9b3e9d0ea..ec53438f52 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,22 @@ +Changes in synapse 0.2.3 (2014-09-12) +===================================== + +Homeserver: + * Fix bug where we stopped sending events to remote home servers if a + user from that home server left, even if there were some still in the + room. + * Fix bugs in the state conflict resolution where it was incorrectly + rejecting events. + +Webclient: + * Display room names and topics. + * Allow setting/editing of room names and topics. + * Display information about rooms on the main page. + * Handle ban and kick events in real time. + * VoIP UI and reliabilty improvements. + * Improvements to initial startup speed. + * Don't display duplicate join events. + Changes in synapse 0.2.2 (2014-09-06) ===================================== diff --git a/VERSION b/VERSION index ee1372d33a..7179039691 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.2 +0.2.3 diff --git a/synapse/__init__.py b/synapse/__init__.py index 1ed9cdcdf3..d60267ebe4 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.2.2" +__version__ = "0.2.3" -- cgit 1.4.1 From 34878bc26a2ed4b796412830a4e1bf9edddc0089 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 15 Sep 2014 10:23:20 +0100 Subject: Added LoginType constants. Created general structure for processing registrations. --- synapse/api/constants.py | 9 +++++ synapse/rest/register.py | 95 ++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 84 insertions(+), 20 deletions(-) (limited to 'synapse') diff --git a/synapse/api/constants.py b/synapse/api/constants.py index fcef062fc9..618d3d7577 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -50,3 +50,12 @@ class JoinRules(object): KNOCK = u"knock" INVITE = u"invite" PRIVATE = u"private" + + +class LoginType(object): + PASSWORD = u"m.login.password" + OAUTH = u"m.login.oauth2" + EMAIL_CODE = u"m.login.email.code" + EMAIL_URL = u"m.login.email.url" + EMAIL_IDENTITY = u"m.login.email.identity" + RECAPTCHA = u"m.login.recaptcha" \ No newline at end of file diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 48d3c6eca0..8faa26572a 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -17,6 +17,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes +from synapse.api.constants import LoginType from base import RestServlet, client_path_pattern import json @@ -26,31 +27,64 @@ import urllib class RegisterRestServlet(RestServlet): PATTERN = client_path_pattern("/register$") + def on_GET(self, request): + return (200, { + "flows": [ + { + "type": LoginType.RECAPTCHA, + "stages": ([LoginType.RECAPTCHA, LoginType.EMAIL_IDENTITY, + LoginType.PASSWORD]) + }, + { + "type": LoginType.RECAPTCHA, + "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD] + }, + ] + }) + @defer.inlineCallbacks def on_POST(self, request): - desired_user_id = None - password = None + register_json = _parse_json(request) + + session = (register_json["session"] if "session" in register_json + else None) try: - register_json = json.loads(request.content.read()) - if "password" in register_json: - password = register_json["password"].encode("utf-8") - - if type(register_json["user_id"]) == unicode: - desired_user_id = register_json["user_id"].encode("utf-8") - if urllib.quote(desired_user_id) != desired_user_id: - raise SynapseError( - 400, - "User ID must only contain characters which do not " + - "require URL encoding.") - except ValueError: - defer.returnValue((400, "No JSON object.")) + login_type = register_json["type"] + stages = { + LoginType.RECAPTCHA: self._do_recaptcha, + LoginType.PASSWORD: self._do_password, + LoginType.EMAIL_IDENTITY: self._do_email_identity + } + + session_info = None + if session: + session_info = self._get_session_info(session) + + response = yield stages[login_type](register_json, session_info) + defer.returnValue((200, response)) except KeyError: - pass # user_id is optional + raise SynapseError(400, "Bad login type.") + + + desired_user_id = None + password = None + + if "password" in register_json: + password = register_json["password"].encode("utf-8") + + if ("user_id" in register_json and + type(register_json["user_id"]) == unicode): + desired_user_id = register_json["user_id"].encode("utf-8") + if urllib.quote(desired_user_id) != desired_user_id: + raise SynapseError( + 400, + "User ID must only contain characters which do not " + + "require URL encoding.") threepidCreds = None if 'threepidCreds' in register_json: threepidCreds = register_json['threepidCreds'] - + captcha = {} if self.hs.config.enable_registration_captcha: challenge = None @@ -65,7 +99,7 @@ class RegisterRestServlet(RestServlet): except KeyError: raise SynapseError(400, "Captcha response is required", errcode=Codes.CAPTCHA_NEEDED) - + # TODO determine the source IP : May be an X-Forwarding-For header depending on config ip_addr = request.getClientIP() if self.hs.config.captcha_ip_origin_is_x_forwarded: @@ -73,14 +107,14 @@ class RegisterRestServlet(RestServlet): if request.requestHeaders.hasHeader("X-Forwarded-For"): ip_addr = request.requestHeaders.getRawHeaders( "X-Forwarded-For")[0] - + captcha = { "ip": ip_addr, "private_key": self.hs.config.recaptcha_private_key, "challenge": challenge, "response": user_response } - + handler = self.handlers.registration_handler (user_id, token) = yield handler.register( @@ -101,6 +135,27 @@ class RegisterRestServlet(RestServlet): def on_OPTIONS(self, request): return (200, {}) + def _get_session_info(self, session_id): + pass + + def _do_recaptcha(self, register_json, session): + pass + + def _do_email_identity(self, register_json, session): + pass + + def _do_password(self, register_json, session): + pass + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.") + return content + except ValueError: + raise SynapseError(400, "Content not JSON.") def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) -- cgit 1.4.1 From 285ecaacd0308f8088e38c64d49cd2e56b514d3d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 15 Sep 2014 12:42:36 +0100 Subject: Split out password/captcha/email logic. --- synapse/handlers/register.py | 120 ++++++++++++---------- synapse/rest/register.py | 237 ++++++++++++++++++++++++++++--------------- 2 files changed, 217 insertions(+), 140 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 0b841d6d3a..a019d770d4 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -40,8 +40,7 @@ class RegistrationHandler(BaseHandler): self.distributor.declare("registered_user") @defer.inlineCallbacks - def register(self, localpart=None, password=None, threepidCreds=None, - captcha_info={}): + def register(self, localpart=None, password=None): """Registers a new client on the server. Args: @@ -54,37 +53,6 @@ class RegistrationHandler(BaseHandler): Raises: RegistrationError if there was a problem registering. """ - if captcha_info: - captcha_response = yield self._validate_captcha( - captcha_info["ip"], - captcha_info["private_key"], - captcha_info["challenge"], - captcha_info["response"] - ) - if not captcha_response["valid"]: - logger.info("Invalid captcha entered from %s. Error: %s", - captcha_info["ip"], captcha_response["error_url"]) - raise InvalidCaptchaError( - error_url=captcha_response["error_url"] - ) - else: - logger.info("Valid captcha entered from %s", captcha_info["ip"]) - - if threepidCreds: - for c in threepidCreds: - logger.info("validating theeepidcred sid %s on id server %s", - c['sid'], c['idServer']) - try: - threepid = yield self._threepid_from_creds(c) - except: - logger.err() - raise RegistrationError(400, "Couldn't validate 3pid") - - if not threepid: - raise RegistrationError(400, "Couldn't validate 3pid") - logger.info("got threepid medium %s address %s", - threepid['medium'], threepid['address']) - password_hash = None if password: password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) @@ -126,15 +94,54 @@ class RegistrationHandler(BaseHandler): raise RegistrationError( 500, "Cannot generate user ID.") - # Now we have a matrix ID, bind it to the threepids we were given - if threepidCreds: - for c in threepidCreds: - # XXX: This should be a deferred list, shouldn't it? - yield self._bind_threepid(c, user_id) - - defer.returnValue((user_id, token)) + @defer.inlineCallbacks + def check_recaptcha(self, ip, private_key, challenge, response): + """Checks a recaptcha is correct.""" + + captcha_response = yield self._validate_captcha( + ip, + private_key, + challenge, + response + ) + if not captcha_response["valid"]: + logger.info("Invalid captcha entered from %s. Error: %s", + ip, captcha_response["error_url"]) + raise InvalidCaptchaError( + error_url=captcha_response["error_url"] + ) + else: + logger.info("Valid captcha entered from %s", ip) + + @defer.inlineCallbacks + def register_email(self, threepidCreds): + """Registers emails with an identity server.""" + + for c in threepidCreds: + logger.info("validating theeepidcred sid %s on id server %s", + c['sid'], c['idServer']) + try: + threepid = yield self._threepid_from_creds(c) + except: + logger.err() + raise RegistrationError(400, "Couldn't validate 3pid") + + if not threepid: + raise RegistrationError(400, "Couldn't validate 3pid") + logger.info("got threepid medium %s address %s", + threepid['medium'], threepid['address']) + + @defer.inlineCallbacks + def bind_emails(self, user_id, threepidCreds): + """Links emails with a user ID and informs an identity server.""" + + # Now we have a matrix ID, bind it to the threepids we were given + for c in threepidCreds: + # XXX: This should be a deferred list, shouldn't it? + yield self._bind_threepid(c, user_id) + def _generate_token(self, user_id): # urlsafe variant uses _ and - so use . as the separator and replace # all =s with .s so http clients don't quote =s when it is used as @@ -149,17 +156,17 @@ class RegistrationHandler(BaseHandler): def _threepid_from_creds(self, creds): httpCli = PlainHttpClient(self.hs) # XXX: make this configurable! - trustedIdServers = [ 'matrix.org:8090' ] + trustedIdServers = ['matrix.org:8090'] if not creds['idServer'] in trustedIdServers: - logger.warn('%s is not a trusted ID server: rejecting 3pid '+ + logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) defer.returnValue(None) data = yield httpCli.get_json( creds['idServer'], "/_matrix/identity/api/v1/3pid/getValidated3pid", - { 'sid': creds['sid'], 'clientSecret': creds['clientSecret'] } + {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} ) - + if 'medium' in data: defer.returnValue(data) defer.returnValue(None) @@ -170,44 +177,45 @@ class RegistrationHandler(BaseHandler): data = yield httpCli.post_urlencoded_get_json( creds['idServer'], "/_matrix/identity/api/v1/3pid/bind", - { 'sid': creds['sid'], 'clientSecret': creds['clientSecret'], - 'mxid':mxid } + {'sid': creds['sid'], 'clientSecret': creds['clientSecret'], + 'mxid': mxid} ) defer.returnValue(data) - + @defer.inlineCallbacks def _validate_captcha(self, ip_addr, private_key, challenge, response): """Validates the captcha provided. - + Returns: dict: Containing 'valid'(bool) and 'error_url'(str) if invalid. - + """ - response = yield self._submit_captcha(ip_addr, private_key, challenge, + response = yield self._submit_captcha(ip_addr, private_key, challenge, response) # parse Google's response. Lovely format.. lines = response.split('\n') json = { "valid": lines[0] == 'true', - "error_url": "http://www.google.com/recaptcha/api/challenge?"+ + "error_url": "http://www.google.com/recaptcha/api/challenge?" + "error=%s" % lines[1] } defer.returnValue(json) - + @defer.inlineCallbacks def _submit_captcha(self, ip_addr, private_key, challenge, response): client = PlainHttpClient(self.hs) data = yield client.post_urlencoded_get_raw( "www.google.com:80", "/recaptcha/api/verify", - accept_partial=True, # twisted dislikes google's response, no content length. - args={ - 'privatekey': private_key, + # twisted dislikes google's response, no content length. + accept_partial=True, + args={ + 'privatekey': private_key, 'remoteip': ip_addr, 'challenge': challenge, 'response': response } ) defer.returnValue(data) - + diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 8faa26572a..8036c3c406 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -19,28 +19,62 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes from synapse.api.constants import LoginType from base import RestServlet, client_path_pattern +import synapse.util.stringutils as stringutils import json +import logging import urllib +logger = logging.getLogger(__name__) + class RegisterRestServlet(RestServlet): + """Handles registration with the home server. + + This servlet is in control of the registration flow; the registration + handler doesn't have a concept of multi-stages or sessions. + """ + PATTERN = client_path_pattern("/register$") + def __init__(self, hs): + super(RegisterRestServlet, self).__init__(hs) + # sessions are stored as: + # self.sessions = { + # "session_id" : { __session_dict__ } + # } + # TODO: persistent storage + self.sessions = {} + def on_GET(self, request): - return (200, { - "flows": [ - { - "type": LoginType.RECAPTCHA, - "stages": ([LoginType.RECAPTCHA, LoginType.EMAIL_IDENTITY, - LoginType.PASSWORD]) - }, - { - "type": LoginType.RECAPTCHA, - "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD] - }, - ] - }) + if self.hs.config.enable_registration_captcha: + return (200, { + "flows": [ + { + "type": LoginType.RECAPTCHA, + "stages": ([LoginType.RECAPTCHA, + LoginType.EMAIL_IDENTITY, + LoginType.PASSWORD]) + }, + { + "type": LoginType.RECAPTCHA, + "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD] + } + ] + }) + else: + return (200, { + "flows": [ + { + "type": LoginType.EMAIL_IDENTITY, + "stages": ([LoginType.EMAIL_IDENTITY, + LoginType.PASSWORD]) + }, + { + "type": LoginType.PASSWORD + } + ] + }) @defer.inlineCallbacks def on_POST(self, request): @@ -56,96 +90,130 @@ class RegisterRestServlet(RestServlet): LoginType.EMAIL_IDENTITY: self._do_email_identity } - session_info = None - if session: - session_info = self._get_session_info(session) + session_info = self._get_session_info(request, session) + logger.debug("%s : session info %s request info %s", + login_type, session_info, register_json) + response = yield stages[login_type]( + request, + register_json, + session_info + ) + + if "access_token" not in response: + # isn't a final response + response["session"] = session_info["id"] - response = yield stages[login_type](register_json, session_info) defer.returnValue((200, response)) - except KeyError: - raise SynapseError(400, "Bad login type.") + except KeyError as e: + logger.exception(e) + raise SynapseError(400, "Missing JSON keys or bad login type.") + def on_OPTIONS(self, request): + return (200, {}) - desired_user_id = None - password = None + def _get_session_info(self, request, session_id): + if not session_id: + # create a new session + while session_id is None or session_id in self.sessions: + session_id = stringutils.random_string(24) + self.sessions[session_id] = { + "id": session_id, + LoginType.EMAIL_IDENTITY: False, + LoginType.RECAPTCHA: False + } - if "password" in register_json: - password = register_json["password"].encode("utf-8") + return self.sessions[session_id] - if ("user_id" in register_json and - type(register_json["user_id"]) == unicode): - desired_user_id = register_json["user_id"].encode("utf-8") - if urllib.quote(desired_user_id) != desired_user_id: - raise SynapseError( - 400, - "User ID must only contain characters which do not " + - "require URL encoding.") + def _save_session(self, session): + # TODO: Persistent storage + logger.debug("Saving session %s", session) + self.sessions[session["id"]] = session - threepidCreds = None - if 'threepidCreds' in register_json: - threepidCreds = register_json['threepidCreds'] + def _remove_session(self, session): + logger.debug("Removing session %s", session) + self.sessions.pop(session["id"]) - captcha = {} - if self.hs.config.enable_registration_captcha: - challenge = None - user_response = None - try: - captcha_type = register_json["captcha"]["type"] - if captcha_type != "m.login.recaptcha": - raise SynapseError(400, "Sorry, only m.login.recaptcha " + - "requests are supported.") - challenge = register_json["captcha"]["challenge"] - user_response = register_json["captcha"]["response"] - except KeyError: - raise SynapseError(400, "Captcha response is required", - errcode=Codes.CAPTCHA_NEEDED) - - # TODO determine the source IP : May be an X-Forwarding-For header depending on config - ip_addr = request.getClientIP() - if self.hs.config.captcha_ip_origin_is_x_forwarded: - # use the header - if request.requestHeaders.hasHeader("X-Forwarded-For"): - ip_addr = request.requestHeaders.getRawHeaders( - "X-Forwarded-For")[0] - - captcha = { - "ip": ip_addr, - "private_key": self.hs.config.recaptcha_private_key, - "challenge": challenge, - "response": user_response - } + def _do_recaptcha(self, request, register_json, session): + if not self.hs.config.enable_registration_captcha: + raise SynapseError(400, "Captcha not required.") + challenge = None + user_response = None + try: + challenge = register_json["challenge"] + user_response = register_json["response"] + except KeyError: + raise SynapseError(400, "Captcha response is required", + errcode=Codes.CAPTCHA_NEEDED) + + # May be an X-Forwarding-For header depending on config + ip_addr = request.getClientIP() + if self.hs.config.captcha_ip_origin_is_x_forwarded: + # use the header + if request.requestHeaders.hasHeader("X-Forwarded-For"): + ip_addr = request.requestHeaders.getRawHeaders( + "X-Forwarded-For")[0] handler = self.handlers.registration_handler + yield handler.check_recaptcha( + ip_addr, + self.hs.config.recaptcha_private_key, + challenge, + user_response + ) + session[LoginType.RECAPTCHA] = True # mark captcha as done + self._save_session(session) + defer.returnValue({ + "next": [LoginType.PASSWORD, LoginType.EMAIL_IDENTITY] + }) + + @defer.inlineCallbacks + def _do_email_identity(self, request, register_json, session): + if (self.hs.config.enable_registration_captcha and + not session[LoginType.RECAPTCHA]): + raise SynapseError(400, "Captcha is required.") + + threepidCreds = register_json['threepidCreds'] + handler = self.handlers.registration_handler + yield handler.register_email(threepidCreds) + session["threepidCreds"] = threepidCreds # store creds for next stage + session[LoginType.EMAIL_IDENTITY] = True # mark email as done + self._save_session(session) + defer.returnValue({ + "next": LoginType.PASSWORD + }) + + @defer.inlineCallbacks + def _do_password(self, request, register_json, session): + if (self.hs.config.enable_registration_captcha and + not session[LoginType.RECAPTCHA]): + # captcha should've been done by this stage! + raise SynapseError(400, "Captcha is required.") + + password = register_json["password"].encode("utf-8") + desired_user_id = (register_json["user_id"].encode("utf-8") if "user_id" + in register_json else None) + if desired_user_id and urllib.quote(desired_user_id) != desired_user_id: + raise SynapseError( + 400, + "User ID must only contain characters which do not " + + "require URL encoding.") + handler = self.handlers.registration_handler (user_id, token) = yield handler.register( localpart=desired_user_id, - password=password, - threepidCreds=threepidCreds, - captcha_info=captcha) + password=password + ) + + if session[LoginType.EMAIL_IDENTITY]: + yield handler.bind_emails(user_id, session["threepidCreds"]) result = { "user_id": user_id, "access_token": token, "home_server": self.hs.hostname, } - defer.returnValue( - (200, result) - ) - - def on_OPTIONS(self, request): - return (200, {}) - - def _get_session_info(self, session_id): - pass - - def _do_recaptcha(self, register_json, session): - pass - - def _do_email_identity(self, register_json, session): - pass - - def _do_password(self, register_json, session): - pass + self._remove_session(session) + defer.returnValue(result) def _parse_json(request): @@ -157,5 +225,6 @@ def _parse_json(request): except ValueError: raise SynapseError(400, "Content not JSON.") + def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) -- cgit 1.4.1 From 5bd9369a62c6d6cf677e9eef7d58096449542cdf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Sep 2014 13:26:05 +0100 Subject: Correctly handle the 'age' key in events and pdus --- synapse/api/events/__init__.py | 13 +++++++++++++ synapse/api/events/factory.py | 8 ++++++++ synapse/federation/replication.py | 15 ++++++++++++--- synapse/handlers/events.py | 10 ++++------ synapse/handlers/message.py | 6 +++--- synapse/handlers/room.py | 2 +- synapse/rest/events.py | 2 +- synapse/rest/room.py | 2 +- synapse/server.py | 4 ++++ synapse/storage/_base.py | 4 ++++ 10 files changed, 51 insertions(+), 15 deletions(-) (limited to 'synapse') diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 5f300de108..72c493db57 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -17,6 +17,18 @@ from synapse.api.errors import SynapseError, Codes from synapse.util.jsonobject import JsonEncodedObject +def serialize_event(hs, e): + # FIXME(erikj): To handle the case of presence events and the like + if not isinstance(e, SynapseEvent): + return e + + d = e.get_dict() + if "age_ts" in d: + d["age"] = int(hs.get_clock().time_msec()) - d["age_ts"] + + return d + + class SynapseEvent(JsonEncodedObject): """Base class for Synapse events. These are JSON objects which must abide @@ -43,6 +55,7 @@ class SynapseEvent(JsonEncodedObject): "content", # HTTP body, JSON "state_key", "required_power_level", + "age_ts", ] internal_keys = [ diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 5e38cdbc44..d3d96d73eb 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -59,6 +59,14 @@ class EventFactory(object): if "ts" not in kwargs: kwargs["ts"] = int(self.clock.time_msec()) + # The "age" key is a delta timestamp that should be converted into an + # absolute timestamp the minute we see it. + if "age" in kwargs: + kwargs["age_ts"] = int(self.clock.time_msec()) - int(kwargs["age"]) + del kwargs["age"] + elif "age_ts" not in kwargs: + kwargs["age_ts"] = int(self.clock.time_msec()) + if etype in self._event_list: handler = self._event_list[etype] else: diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index e12510017f..c79ce44688 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -291,6 +291,12 @@ class ReplicationLayer(object): def on_incoming_transaction(self, transaction_data): transaction = Transaction(**transaction_data) + for p in transaction.pdus: + if "age" in p: + p["age_ts"] = int(self.clock.time_msec()) - int(p["age"]) + + pdu_list = [Pdu(**p) for p in transaction.pdus] + logger.debug("[%s] Got transaction", transaction.transaction_id) response = yield self.transaction_actions.have_responded(transaction) @@ -303,8 +309,6 @@ class ReplicationLayer(object): logger.debug("[%s] Transacition is new", transaction.transaction_id) - pdu_list = [Pdu(**p) for p in transaction.pdus] - dl = [] for pdu in pdu_list: dl.append(self._handle_new_pdu(pdu)) @@ -405,9 +409,14 @@ class ReplicationLayer(object): """Returns a new Transaction containing the given PDUs suitable for transmission. """ + pdus = [p.get_dict() for p in pdu_list] + for p in pdus: + if "age_ts" in pdus: + p["age"] = int(self.clock.time_msec()) - p["age_ts"] + return Transaction( - pdus=[p.get_dict() for p in pdu_list], origin=self.server_name, + pdus=pdus, ts=int(self._clock.time_msec()), destination=None, ) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index fd24a11fb8..93dcd40324 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -15,7 +15,6 @@ from twisted.internet import defer -from synapse.api.events import SynapseEvent from synapse.util.logutils import log_function from ._base import BaseHandler @@ -71,10 +70,7 @@ class EventStreamHandler(BaseHandler): auth_user, room_ids, pagin_config, timeout ) - chunks = [ - e.get_dict() if isinstance(e, SynapseEvent) else e - for e in events - ] + chunks = [self.hs.serialize_event(e) for e in events] chunk = { "chunk": chunks, @@ -92,7 +88,9 @@ class EventStreamHandler(BaseHandler): # 10 seconds of grace to allow the client to reconnect again # before we think they're gone def _later(): - logger.debug("_later stopped_user_eventstream %s", auth_user) + logger.debug( + "_later stopped_user_eventstream %s", auth_user + ) self.distributor.fire( "stopped_user_eventstream", auth_user ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 87fc04478b..b63863e5b2 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -124,7 +124,7 @@ class MessageHandler(BaseHandler): ) chunk = { - "chunk": [e.get_dict() for e in events], + "chunk": [self.hs.serialize_event(e) for e in events], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } @@ -296,7 +296,7 @@ class MessageHandler(BaseHandler): end_token = now_token.copy_and_replace("room_key", token[1]) d["messages"] = { - "chunk": [m.get_dict() for m in messages], + "chunk": [self.hs.serialize_event(m) for m in messages], "start": start_token.to_string(), "end": end_token.to_string(), } @@ -304,7 +304,7 @@ class MessageHandler(BaseHandler): current_state = yield self.store.get_current_state( event.room_id ) - d["state"] = [c.get_dict() for c in current_state] + d["state"] = [self.hs.serialize_event(c) for c in current_state] except: logger.exception("Failed to get snapshot") diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 310cb46fe7..5bc1280432 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -335,7 +335,7 @@ class RoomMemberHandler(BaseHandler): member_list = yield self.store.get_room_members(room_id=room_id) event_list = [ - entry.get_dict() + self.hs.serialize_event(entry) for entry in member_list ] chunk_data = { diff --git a/synapse/rest/events.py b/synapse/rest/events.py index 7fde143200..097195d7cc 100644 --- a/synapse/rest/events.py +++ b/synapse/rest/events.py @@ -59,7 +59,7 @@ class EventRestServlet(RestServlet): event = yield handler.get_event(auth_user, event_id) if event: - defer.returnValue((200, event.get_dict())) + defer.returnValue((200, self.hs.serialize_event(event))) else: defer.returnValue((404, "Event not found.")) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index cef700c81c..ecb1e346d9 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -378,7 +378,7 @@ class RoomTriggerBackfill(RestServlet): handler = self.handlers.federation_handler events = yield handler.backfill(remote_server, room_id, limit) - res = [event.get_dict() for event in events] + res = [self.hs.serialize_event(event) for event in events] defer.returnValue((200, res)) diff --git a/synapse/server.py b/synapse/server.py index 83368ea5a7..7c185537aa 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -20,6 +20,7 @@ # Imports required for the default HomeServer() implementation from synapse.federation import initialize_http_replication +from synapse.api.events import serialize_event from synapse.api.events.factory import EventFactory from synapse.notifier import Notifier from synapse.api.auth import Auth @@ -138,6 +139,9 @@ class BaseHomeServer(object): object.""" return RoomID.from_string(s, hs=self) + def serialize_event(self, e): + return serialize_event(self, e) + # Build magic accessors for every dependency for depname in BaseHomeServer.DEPENDENCIES: BaseHomeServer._make_dependency_method(depname) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 8deaaf93bd..cf88bfc22b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -315,6 +315,10 @@ class SQLBaseStore(object): d["content"] = json.loads(d["content"]) del d["unrecognized_keys"] + if "age_ts" not in d: + # For compatibility + d["age_ts"] = d["ts"] if "ts" in d else 0 + return self.event_factory.create_event( etype=d["type"], **d -- cgit 1.4.1 From 04fbda46ddb722f64f9b0d702b41cc22569e9b12 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 15 Sep 2014 14:52:39 +0100 Subject: Make captcha work again with the new registration logic. --- synapse/rest/register.py | 1 + webclient/components/matrix/matrix-service.js | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 8036c3c406..fe8f0ed23f 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -133,6 +133,7 @@ class RegisterRestServlet(RestServlet): logger.debug("Removing session %s", session) self.sessions.pop(session["id"]) + @defer.inlineCallbacks def _do_recaptcha(self, request, register_json, session): if not self.hs.config.enable_registration_captcha: raise SynapseError(400, "Captcha not required.") diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js index d7d278a7f6..35ebca961c 100644 --- a/webclient/components/matrix/matrix-service.js +++ b/webclient/components/matrix/matrix-service.js @@ -154,6 +154,13 @@ angular.module('matrixService', []) } if (!useCaptcha && regType == "m.login.recaptcha") { console.error("Web client setup to not use captcha, but HS demands a captcha."); + deferred.reject({ + data: { + errcode: "M_CAPTCHA_NEEDED", + error: "Home server requires a captcha." + } + }); + return; } } } @@ -183,7 +190,20 @@ angular.module('matrixService', []) deferred.resolve(response); } else if (response.data.next) { - return doRegisterLogin(path, response.data.next, sessionId, user_name, password, threepidCreds).then( + var nextType = response.data.next; + if (response.data.next instanceof Array) { + for (var i=0; i Date: Mon, 15 Sep 2014 15:38:29 +0100 Subject: Be consistent when associating keys with login types for registration/login. --- cmdclient/console.py | 2 +- synapse/rest/register.py | 2 +- tests/rest/utils.py | 2 +- webclient/components/matrix/matrix-service.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/cmdclient/console.py b/cmdclient/console.py index 5a9d4c3c4c..d9c6ec6a70 100755 --- a/cmdclient/console.py +++ b/cmdclient/console.py @@ -162,7 +162,7 @@ class SynapseCmd(cmd.Cmd): "type": "m.login.password" } if "userid" in args: - body["user_id"] = args["userid"] + body["user"] = args["userid"] if password: body["password"] = password diff --git a/synapse/rest/register.py b/synapse/rest/register.py index fe8f0ed23f..c2c80e70c7 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -192,7 +192,7 @@ class RegisterRestServlet(RestServlet): raise SynapseError(400, "Captcha is required.") password = register_json["password"].encode("utf-8") - desired_user_id = (register_json["user_id"].encode("utf-8") if "user_id" + desired_user_id = (register_json["user"].encode("utf-8") if "user" in register_json else None) if desired_user_id and urllib.quote(desired_user_id) != desired_user_id: raise SynapseError( diff --git a/tests/rest/utils.py b/tests/rest/utils.py index 25ed1388cf..579441fb4a 100644 --- a/tests/rest/utils.py +++ b/tests/rest/utils.py @@ -99,7 +99,7 @@ class RestTestCase(unittest.TestCase): "POST", "/register", json.dumps({ - "user_id": user_id, + "user": user_id, "password": "test", "type": "m.login.password" })) diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js index 35ebca961c..069e02e939 100644 --- a/webclient/components/matrix/matrix-service.js +++ b/webclient/components/matrix/matrix-service.js @@ -100,7 +100,7 @@ angular.module('matrixService', []) } else if (loginType === "m.login.password") { data = { - user_id: userName, + user: userName, password: password }; } -- cgit 1.4.1 From 34d7896b06ba72c4a7ea28d5c42124a35df121bd Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 15 Sep 2014 16:05:51 +0100 Subject: More helpful 400 error messages. --- synapse/rest/register.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/register.py b/synapse/rest/register.py index c2c80e70c7..af528a44f6 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -82,6 +82,10 @@ class RegisterRestServlet(RestServlet): session = (register_json["session"] if "session" in register_json else None) + login_type = None + if "type" not in register_json: + raise SynapseError(400, "Missing 'type' key.") + try: login_type = register_json["type"] stages = { @@ -106,7 +110,7 @@ class RegisterRestServlet(RestServlet): defer.returnValue((200, response)) except KeyError as e: logger.exception(e) - raise SynapseError(400, "Missing JSON keys or bad login type.") + raise SynapseError(400, "Missing JSON keys for login type %s." % login_type) def on_OPTIONS(self, request): return (200, {}) -- cgit 1.4.1 From 6ac0b4ade86d1bdb59c01ff8edff6b149cf1981e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Sep 2014 14:54:25 +0100 Subject: Fix 'age' key to update on retries --- synapse/federation/replication.py | 19 ++++++++++++++++--- synapse/federation/transport.py | 17 +++++++++++++++-- synapse/http/client.py | 13 ++++++++++--- 3 files changed, 41 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index c79ce44688..a48a7ac15f 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -292,8 +292,8 @@ class ReplicationLayer(object): transaction = Transaction(**transaction_data) for p in transaction.pdus: - if "age" in p: - p["age_ts"] = int(self.clock.time_msec()) - int(p["age"]) + if "age_ts" in p: + p["age"] = int(self._clock.time_msec()) - int(p["age_ts"]) pdu_list = [Pdu(**p) for p in transaction.pdus] @@ -602,8 +602,21 @@ class _TransactionQueue(object): logger.debug("TX [%s] Sending transaction...", destination) # Actually send the transaction + + # FIXME (erikj): This is a bit of a hack to make the Pdu age + # keys work + def cb(transaction): + now = int(self._clock.time_msec()) + if "pdus" in transaction: + for p in transaction["pdus"]: + if "age_ts" in p: + p["age"] = now - int(p["age_ts"]) + + return transaction + code, response = yield self.transport_layer.send_transaction( - transaction + transaction, + on_send_callback=cb, ) logger.debug("TX [%s] Sent transaction", destination) diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 6e62ae7c74..afc777ec9e 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -144,7 +144,7 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function - def send_transaction(self, transaction): + def send_transaction(self, transaction, on_send_callback=None): """ Sends the given Transaction to it's destination Args: @@ -165,10 +165,23 @@ class TransportLayer(object): data = transaction.get_dict() + # FIXME (erikj): This is a bit of a hack to make the Pdu age + # keys work + def cb(destination, method, path_bytes, producer): + if not on_send_callback: + return + + transaction = json.loads(producer.body) + + new_transaction = on_send_callback(transaction) + + producer.reset(new_transaction) + code, response = yield self.client.put_json( transaction.destination, path=PREFIX + "/send/%s/" % transaction.transaction_id, - data=data + data=data, + on_send_callback=cb, ) logger.debug( diff --git a/synapse/http/client.py b/synapse/http/client.py index ece6318e00..eb11bfd4d5 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -122,7 +122,7 @@ class TwistedHttpClient(HttpClient): self.hs = hs @defer.inlineCallbacks - def put_json(self, destination, path, data): + def put_json(self, destination, path, data, on_send_callback=None): if destination in _destination_mappings: destination = _destination_mappings[destination] @@ -131,7 +131,8 @@ class TwistedHttpClient(HttpClient): "PUT", path.encode("ascii"), producer=_JsonProducer(data), - headers_dict={"Content-Type": ["application/json"]} + headers_dict={"Content-Type": ["application/json"]}, + on_send_callback=on_send_callback, ) logger.debug("Getting resp body") @@ -218,7 +219,7 @@ class TwistedHttpClient(HttpClient): @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, param_bytes=b"", query_bytes=b"", producer=None, headers_dict={}, - retry_on_dns_fail=True): + retry_on_dns_fail=True, on_send_callback=None): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [b"Synapse"] @@ -242,6 +243,9 @@ class TwistedHttpClient(HttpClient): endpoint = self._getEndpoint(reactor, destination); while True: + if on_send_callback: + on_send_callback(destination, method, path_bytes, producer) + try: response = yield self.agent.request( destination, @@ -310,6 +314,9 @@ class _JsonProducer(object): """ Used by the twisted http client to create the HTTP body from json """ def __init__(self, jsn): + self.reset(jsn) + + def reset(self, jsn): self.body = encode_canonical_json(jsn) self.length = len(self.body) -- cgit 1.4.1 From e639a3516d271c395862bcd0c6facfd8c5c9ff58 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Sep 2014 15:18:51 +0100 Subject: Improve logging in federation handler. --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 59cbf71d78..5187bcb5bb 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -93,6 +93,8 @@ class FederationHandler(BaseHandler): """ event = self.pdu_codec.event_from_pdu(pdu) + logger.debug("Got event: %s", event.event_id) + with (yield self.lock_manager.lock(pdu.context)): if event.is_state and not backfilled: is_new_state = yield self.state_handler.handle_new_state( @@ -106,7 +108,7 @@ class FederationHandler(BaseHandler): # respond to PDU. if hasattr(event, "state_key") and not is_new_state: - logger.debug("Ignoring old state.") + logger.debug("Ignoring old state: %s", event.event_id) return target_is_mine = False -- cgit 1.4.1 From 59516a8bb1cd8040bd07420f84b856bd8904d6c8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Sep 2014 16:40:44 +0100 Subject: Correctly handle receiving 'missing' Pdus from federation, rather than just discarding them. --- synapse/handlers/federation.py | 12 +++++------- synapse/storage/__init__.py | 15 ++++++++++----- tests/handlers/test_federation.py | 4 +++- 3 files changed, 18 insertions(+), 13 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5187bcb5bb..001c6c110c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -100,17 +100,11 @@ class FederationHandler(BaseHandler): is_new_state = yield self.state_handler.handle_new_state( pdu ) - if not is_new_state: - return else: is_new_state = False # TODO: Implement something in federation that allows us to # respond to PDU. - if hasattr(event, "state_key") and not is_new_state: - logger.debug("Ignoring old state: %s", event.event_id) - return - target_is_mine = False if hasattr(event, "target_host"): target_is_mine = event.target_host == self.hs.hostname @@ -141,7 +135,11 @@ class FederationHandler(BaseHandler): else: with (yield self.room_lock.lock(event.room_id)): - yield self.store.persist_event(event, backfilled) + yield self.store.persist_event( + event, + backfilled, + is_new_state=is_new_state + ) room = yield self.store.get_room(event.room_id) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9201a377b6..1cede2809d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -68,7 +68,8 @@ class DataStore(RoomMemberStore, RoomStore, @defer.inlineCallbacks @log_function - def persist_event(self, event=None, backfilled=False, pdu=None): + def persist_event(self, event=None, backfilled=False, pdu=None, + is_new_state=True): stream_ordering = None if backfilled: if not self.min_token_deferred.called: @@ -83,6 +84,7 @@ class DataStore(RoomMemberStore, RoomStore, event=event, backfilled=backfilled, stream_ordering=stream_ordering, + is_new_state=is_new_state, ) except _RollbackButIsFineException as e: pass @@ -109,12 +111,14 @@ class DataStore(RoomMemberStore, RoomStore, defer.returnValue(event) def _persist_pdu_event_txn(self, txn, pdu=None, event=None, - backfilled=False, stream_ordering=None): + backfilled=False, stream_ordering=None, + is_new_state=True): if pdu is not None: self._persist_event_pdu_txn(txn, pdu) if event is not None: return self._persist_event_txn( - txn, event, backfilled, stream_ordering + txn, event, backfilled, stream_ordering, + is_new_state=is_new_state, ) def _persist_event_pdu_txn(self, txn, pdu): @@ -141,7 +145,8 @@ class DataStore(RoomMemberStore, RoomStore, self._update_min_depth_for_context_txn(txn, pdu.context, pdu.depth) @log_function - def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None): + def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None, + is_new_state=True): if event.type == RoomMemberEvent.TYPE: self._store_room_member_txn(txn, event) elif event.type == FeedbackEvent.TYPE: @@ -195,7 +200,7 @@ class DataStore(RoomMemberStore, RoomStore, ) raise _RollbackButIsFineException("_persist_event") - if not backfilled and hasattr(event, "state_key"): + if is_new_state and hasattr(event, "state_key"): vals = { "event_id": event.event_id, "room_id": event.room_id, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index f0308a29d3..eb6b7c22ef 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -74,7 +74,9 @@ class FederationTestCase(unittest.TestCase): yield self.handlers.federation_handler.on_receive_pdu(pdu, False) - self.datastore.persist_event.assert_called_once_with(ANY, False) + self.datastore.persist_event.assert_called_once_with( + ANY, False, is_new_state=False + ) self.notifier.on_new_room_event.assert_called_once_with(ANY) @defer.inlineCallbacks -- cgit 1.4.1 From 40d2f38abe604525fb03622995c377904f1ea3dd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Sep 2014 16:55:39 +0100 Subject: Fix bug where we incorrectly calculated 'age_ts' from 'age' key rather than the reverse. Don't transmit age_ts to clients for now. --- synapse/api/events/__init__.py | 1 + synapse/federation/replication.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 72c493db57..add81ec3e6 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -25,6 +25,7 @@ def serialize_event(hs, e): d = e.get_dict() if "age_ts" in d: d["age"] = int(hs.get_clock().time_msec()) - d["age_ts"] + del d["age_ts"] return d diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index a48a7ac15f..96b82f00cb 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -292,8 +292,9 @@ class ReplicationLayer(object): transaction = Transaction(**transaction_data) for p in transaction.pdus: - if "age_ts" in p: - p["age"] = int(self._clock.time_msec()) - int(p["age_ts"]) + if "age" in p: + p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) + del p["age"] pdu_list = [Pdu(**p) for p in transaction.pdus] -- cgit 1.4.1 From 1e4b971f95ac953e9fbd4a8e4cc0d0d2edc5e5ea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Sep 2014 17:43:46 +0100 Subject: Fix bug where we didn't always get 'prev_content' key --- synapse/api/events/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index add81ec3e6..a9991e9c94 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -57,6 +57,7 @@ class SynapseEvent(JsonEncodedObject): "state_key", "required_power_level", "age_ts", + "prev_content", ] internal_keys = [ @@ -172,10 +173,6 @@ class SynapseEvent(JsonEncodedObject): class SynapseStateEvent(SynapseEvent): - valid_keys = SynapseEvent.valid_keys + [ - "prev_content", - ] - def __init__(self, **kwargs): if "state_key" not in kwargs: kwargs["state_key"] = "" -- cgit 1.4.1 From 5f30a69a9e617028c39ea3851b9a5de43d42a299 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 16 Sep 2014 11:22:40 +0100 Subject: Added PasswordResetRestServlet. Hit the IS to confirm the email/user. Need to send email. --- synapse/handlers/login.py | 29 ++++++++++++++++++++++++++++- synapse/rest/login.py | 22 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 6ee7ce5a2d..101b9a81ad 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -17,9 +17,11 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import LoginError, Codes +from synapse.http.client import PlainHttpClient import bcrypt import logging +import urllib logger = logging.getLogger(__name__) @@ -62,4 +64,29 @@ class LoginHandler(BaseHandler): defer.returnValue(token) else: logger.warn("Failed password login for user %s", user) - raise LoginError(403, "", errcode=Codes.FORBIDDEN) \ No newline at end of file + raise LoginError(403, "", errcode=Codes.FORBIDDEN) + + @defer.inlineCallbacks + def reset_password(self, user_id, email): + is_valid = yield self._check_valid_association(user_id, email) + logger.info("reset_password user=%s email=%s valid=%s", user_id, email, + is_valid) + + @defer.inlineCallbacks + def _check_valid_association(self, user_id, email): + identity = yield self._query_email(email) + if identity and "mxid" in identity: + if identity["mxid"] == user_id: + defer.returnValue(True) + return + defer.returnValue(False) + + @defer.inlineCallbacks + def _query_email(self, email): + httpCli = PlainHttpClient(self.hs) + data = yield httpCli.get_json( + 'matrix.org:8090', # TODO FIXME This should be configurable. + "/_matrix/identity/api/v1/lookup?medium=email&address=" + + "%s" % urllib.quote(email) + ) + defer.returnValue(data) \ No newline at end of file diff --git a/synapse/rest/login.py b/synapse/rest/login.py index ba49afcaa7..7ab9cb51e8 100644 --- a/synapse/rest/login.py +++ b/synapse/rest/login.py @@ -73,6 +73,27 @@ class LoginFallbackRestServlet(RestServlet): return (200, {}) +class PasswordResetRestServlet(RestServlet): + PATTERN = client_path_pattern("/login/reset") + + @defer.inlineCallbacks + def on_POST(self, request): + reset_info = _parse_json(request) + try: + email = reset_info["email"] + user_id = reset_info["user_id"] + handler = self.handlers.login_handler + yield handler.reset_password(user_id, email) + # purposefully give no feedback to avoid people hammering different + # combinations. + defer.returnValue((200, {})) + except KeyError: + raise SynapseError( + 400, + "Missing keys. Requires 'email' and 'user_id'." + ) + + def _parse_json(request): try: content = json.loads(request.content.read()) @@ -85,3 +106,4 @@ def _parse_json(request): def register_servlets(hs, http_server): LoginRestServlet(hs).register(http_server) + PasswordResetRestServlet(hs).register(http_server) -- cgit 1.4.1 From cc83b06cd19f8fc52f86700c1663185a2b1a7cac Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 16 Sep 2014 12:36:39 +0100 Subject: Added support for the HS to send emails. Use it to send password resets. Added email_smtp_server and email_from_address config args. Added emailutils. --- synapse/config/email.py | 39 ++++++++++++++++++++++++ synapse/config/homeserver.py | 8 +++-- synapse/handlers/login.py | 14 +++++++++ synapse/util/emailutils.py | 71 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 synapse/config/email.py create mode 100644 synapse/util/emailutils.py (limited to 'synapse') diff --git a/synapse/config/email.py b/synapse/config/email.py new file mode 100644 index 0000000000..9bcc5a8fea --- /dev/null +++ b/synapse/config/email.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class EmailConfig(Config): + + def __init__(self, args): + super(EmailConfig, self).__init__(args) + self.email_from_address = args.email_from_address + self.email_smtp_server = args.email_smtp_server + + @classmethod + def add_arguments(cls, parser): + super(EmailConfig, cls).add_arguments(parser) + email_group = parser.add_argument_group("email") + email_group.add_argument( + "--email-from-address", + default="FROM@EXAMPLE.COM", + help="The address to send emails from (e.g. for password resets)." + ) + email_group.add_argument( + "--email-smtp-server", + default="", + help="The SMTP server to send emails from (e.g. for password resets)." + ) \ No newline at end of file diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index e16f2c733b..4b810a2302 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -20,11 +20,15 @@ from .database import DatabaseConfig from .ratelimiting import RatelimitConfig from .repository import ContentRepositoryConfig from .captcha import CaptchaConfig +from .email import EmailConfig + class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, - RatelimitConfig, ContentRepositoryConfig, CaptchaConfig): + RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, + EmailConfig): pass -if __name__=='__main__': + +if __name__ == '__main__': import sys HomeServerConfig.load_config("Generate config", sys.argv[1:], "HomeServer") diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 101b9a81ad..80ffdd2726 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -18,6 +18,8 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import LoginError, Codes from synapse.http.client import PlainHttpClient +from synapse.util.emailutils import EmailException +import synapse.util.emailutils as emailutils import bcrypt import logging @@ -71,6 +73,18 @@ class LoginHandler(BaseHandler): is_valid = yield self._check_valid_association(user_id, email) logger.info("reset_password user=%s email=%s valid=%s", user_id, email, is_valid) + if is_valid: + try: + # send an email out + emailutils.send_email( + smtp_server=self.hs.config.email_smtp_server, + from_addr=self.hs.config.email_from_address, + to_addr=email, + subject="Password Reset", + body="TODO." + ) + except EmailException as e: + logger.exception(e) @defer.inlineCallbacks def _check_valid_association(self, user_id, email): diff --git a/synapse/util/emailutils.py b/synapse/util/emailutils.py new file mode 100644 index 0000000000..cdb0abd7ea --- /dev/null +++ b/synapse/util/emailutils.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" This module allows you to send out emails. +""" +import email.utils +import smtplib +import twisted.python.log +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart + +import logging + +logger = logging.getLogger(__name__) + + +class EmailException(Exception): + pass + + +def send_email(smtp_server, from_addr, to_addr, subject, body): + """Sends an email. + + Args: + smtp_server(str): The SMTP server to use. + from_addr(str): The address to send from. + to_addr(str): The address to send to. + subject(str): The subject of the email. + body(str): The plain text body of the email. + Raises: + EmailException if there was a problem sending the mail. + """ + if not smtp_server or not from_addr or not to_addr: + raise EmailException("Need SMTP server, from and to addresses. Check " + + "the config to set these.") + + msg = MIMEMultipart('alternative') + msg['Subject'] = subject + msg['From'] = from_addr + msg['To'] = to_addr + plain_part = MIMEText(body) + msg.attach(plain_part) + + raw_from = email.utils.parseaddr(from_addr)[1] + raw_to = email.utils.parseaddr(to_addr)[1] + if not raw_from or not raw_to: + raise EmailException("Couldn't parse from/to address.") + + logger.info("Sending email to %s on server %s with subject %s", + to_addr, smtp_server, subject) + + try: + smtp = smtplib.SMTP(smtp_server) + smtp.sendmail(raw_from, raw_to, msg.as_string()) + smtp.quit() + except Exception as origException: + twisted.python.log.err() + ese = EmailException() + ese.cause = origException + raise ese \ No newline at end of file -- cgit 1.4.1 From c099b36af3b7d842c86ea56edccacbf1082f25bb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 16 Sep 2014 13:32:33 +0100 Subject: Comment out password reset for now, until the mechanism is fully discussed (IS token auth vs HS auth) --- synapse/rest/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/login.py b/synapse/rest/login.py index 7ab9cb51e8..ad71f6c61d 100644 --- a/synapse/rest/login.py +++ b/synapse/rest/login.py @@ -106,4 +106,4 @@ def _parse_json(request): def register_servlets(hs, http_server): LoginRestServlet(hs).register(http_server) - PasswordResetRestServlet(hs).register(http_server) + # TODO PasswordResetRestServlet(hs).register(http_server) -- cgit 1.4.1 From b6818fd4d2108543ce86b5d8bf540e74a89cf27e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 17 Sep 2014 15:05:09 +0100 Subject: SYN-40: When a user updates their displayname or avatar update all their join events for all the rooms they are currently in. --- synapse/handlers/profile.py | 46 ++++++++++++++++++++++++++++++++++--- tests/handlers/test_presencelike.py | 11 +++++++++ tests/handlers/test_profile.py | 21 +++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 023d8c0cf2..dab9b03f04 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -15,9 +15,9 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, AuthError - -from synapse.api.errors import CodeMessageException +from synapse.api.errors import SynapseError, AuthError, CodeMessageException +from synapse.api.constants import Membership +from synapse.api.events.room import RoomMemberEvent from ._base import BaseHandler @@ -97,6 +97,8 @@ class ProfileHandler(BaseHandler): } ) + yield self._update_join_states(target_user) + @defer.inlineCallbacks def get_avatar_url(self, target_user): if target_user.is_mine: @@ -144,6 +146,8 @@ class ProfileHandler(BaseHandler): } ) + yield self._update_join_states(target_user) + @defer.inlineCallbacks def collect_presencelike_data(self, user, state): if not user.is_mine: @@ -180,3 +184,39 @@ class ProfileHandler(BaseHandler): ) defer.returnValue(response) + + @defer.inlineCallbacks + def _update_join_states(self, user): + if not user.is_mine: + return + + joins = yield self.store.get_rooms_for_user_where_membership_is( + user.to_string(), + [Membership.JOIN], + ) + + for j in joins: + snapshot = yield self.store.snapshot_room( + j.room_id, j.state_key, RoomMemberEvent.TYPE, + j.state_key + ) + + content = { + "membership": j.content["membership"], + "prev": j.content["membership"], + } + + yield self.distributor.fire( + "collect_presencelike_data", user, content + ) + + new_event = self.event_factory.create_event( + etype=j.type, + room_id=j.room_id, + state_key=j.state_key, + content=content, + user_id=j.state_key, + ) + + yield self.state_handler.handle_new_event(new_event, snapshot) + yield self._on_new_room_event(new_event, snapshot) diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 72c55b3667..047752ad68 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -65,6 +65,8 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): "is_presence_visible", "set_profile_displayname", + + "get_rooms_for_user_where_membership_is", ]), handlers=None, resource_for_federation=Mock(), @@ -132,6 +134,10 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): # Remote user self.u_potato = hs.parse_userid("@potato:remote") + self.mock_get_joined = ( + self.datastore.get_rooms_for_user_where_membership_is + ) + @defer.inlineCallbacks def test_set_my_state(self): self.presence_list = [ @@ -152,6 +158,11 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): @defer.inlineCallbacks def test_push_local(self): + def get_joined(*args): + return defer.succeed([]) + + self.mock_get_joined.side_effect = get_joined + self.presence_list = [ {"observed_user_id": "@banana:test"}, {"observed_user_id": "@clementine:test"}, diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 0a5cebb4cc..ee2be9b6d5 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -22,6 +22,7 @@ from mock import Mock from synapse.api.errors import AuthError from synapse.server import HomeServer from synapse.handlers.profile import ProfileHandler +from synapse.api.constants import Membership class ProfileHandlers(object): @@ -50,6 +51,7 @@ class ProfileTestCase(unittest.TestCase): "set_profile_displayname", "get_profile_avatar_url", "set_profile_avatar_url", + "get_rooms_for_user_where_membership_is", ]), handlers=None, resource_for_federation=Mock(), @@ -65,6 +67,10 @@ class ProfileTestCase(unittest.TestCase): self.handler = hs.get_handlers().profile_handler + self.mock_get_joined = ( + self.datastore.get_rooms_for_user_where_membership_is + ) + # TODO(paul): Icky signal declarings.. booo hs.get_distributor().declare("changed_presencelike_data") @@ -83,8 +89,15 @@ class ProfileTestCase(unittest.TestCase): mocked_set = self.datastore.set_profile_displayname mocked_set.return_value = defer.succeed(()) + self.mock_get_joined.return_value = defer.succeed([]) + yield self.handler.set_displayname(self.frank, self.frank, "Frank Jr.") + self.mock_get_joined.assert_called_once_with( + self.frank.to_string(), + [Membership.JOIN] + ) + mocked_set.assert_called_with("1234ABCD", "Frank Jr.") @defer.inlineCallbacks @@ -135,7 +148,15 @@ class ProfileTestCase(unittest.TestCase): mocked_set = self.datastore.set_profile_avatar_url mocked_set.return_value = defer.succeed(()) + self.mock_get_joined.return_value = defer.succeed([]) + yield self.handler.set_avatar_url(self.frank, self.frank, "http://my.server/pic.gif") + self.mock_get_joined.assert_called_once_with( + self.frank.to_string(), + [Membership.JOIN] + ) + + mocked_set.assert_called_with("1234ABCD", "http://my.server/pic.gif") -- cgit 1.4.1 From 9973298e2ac4039b96e923faa984b400ea720b7f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 17 Sep 2014 15:27:45 +0100 Subject: Print expected-vs-actual data types on typecheck failure from check_json() --- synapse/api/events/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 72c493db57..4fe0608016 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -154,7 +154,8 @@ class SynapseEvent(JsonEncodedObject): return "Missing %s key" % key if type(content[key]) != type(template[key]): - return "Key %s is of the wrong type." % key + return "Key %s is of the wrong type (got %s, want %s)" % ( + key, type(content[key]), type(template[key])) if type(content[key]) == dict: # we must go deeper -- cgit 1.4.1 From c707b7d12895c01b998c4676a257f0f5f75a56ba Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 17 Sep 2014 16:05:30 +0100 Subject: SYWEB-3 : Added 'visibility' key to rooms returned via /initialSync --- synapse/handlers/message.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b63863e5b2..14fae689f2 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -268,6 +268,9 @@ class MessageHandler(BaseHandler): user, pagination_config, None ) + public_rooms = yield self.store.get_rooms(is_public=True) + public_room_ids = [r["room_id"] for r in public_rooms] + limit = pagin_config.limit if not limit: limit = 10 @@ -276,6 +279,8 @@ class MessageHandler(BaseHandler): d = { "room_id": event.room_id, "membership": event.membership, + "visibility": ("public" if event.room_id in + public_room_ids else "private"), } if event.membership == Membership.INVITE: -- cgit 1.4.1 From 10b4291b546bce158afb526741b0230b15ac8adf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 17 Sep 2014 17:49:01 +0100 Subject: Bump versions --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/VERSION b/VERSION index 7179039691..0d91a54c7d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.3 +0.3.0 diff --git a/synapse/__init__.py b/synapse/__init__.py index d60267ebe4..8ef176ea6f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.2.3" +__version__ = "0.3.0" -- cgit 1.4.1 From 9fd0c74e90df6fb34f07c526c29d28387fde007f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 18 Sep 2014 14:46:23 +0100 Subject: Bump changelog and versions --- CHANGES.rst | 9 +++++++++ VERSION | 2 +- synapse/__init__.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/CHANGES.rst b/CHANGES.rst index 4e536bc4de..edf32db752 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,12 @@ +Changes in synapse 0.3.1 (2014-09-18) +===================================== +This is a release to hotfix v0.3.0 to fix two regressions. + +Webclient: + * Fix a regression where we sometimes displayed duplicate events. + * Fix a regression where we didn't immediately remove rooms you were + banned in from the recents list. + Changes in synapse 0.3.0 (2014-09-18) ===================================== See UPGRADE for information about changes to the client server API, including diff --git a/VERSION b/VERSION index 0d91a54c7d..9e11b32fca 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 +0.3.1 diff --git a/synapse/__init__.py b/synapse/__init__.py index 8ef176ea6f..1b49cbb38e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.3.0" +__version__ = "0.3.1" -- cgit 1.4.1 From 380852b58e6ab96a8d186bbfe3d95ac9284ceaf3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 18 Sep 2014 16:20:53 +0100 Subject: Bump Changelog and version --- CHANGES.rst | 7 +++++++ VERSION | 2 +- synapse/__init__.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/CHANGES.rst b/CHANGES.rst index edf32db752..9e884e51bf 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +Changes in synapse 0.3.2 (2014-09-18) +===================================== + +Webclient: + * Fix bug where an empty "bing words" list in old accounts didn't send + notifications when it should have done. + Changes in synapse 0.3.1 (2014-09-18) ===================================== This is a release to hotfix v0.3.0 to fix two regressions. diff --git a/VERSION b/VERSION index 9e11b32fca..d15723fbe8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.1 +0.3.2 diff --git a/synapse/__init__.py b/synapse/__init__.py index 1b49cbb38e..0dbad9a0de 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.3.1" +__version__ = "0.3.2" -- cgit 1.4.1 From 3fa01be9e46b338614b7f3d543372902a3484c21 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 19 Sep 2014 12:04:17 +0100 Subject: formatting --- synapse/config/captcha.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index a97a5bab1e..8ebcfc3623 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -14,13 +14,16 @@ from ._base import Config + class CaptchaConfig(Config): def __init__(self, args): super(CaptchaConfig, self).__init__(args) self.recaptcha_private_key = args.recaptcha_private_key self.enable_registration_captcha = args.enable_registration_captcha - self.captcha_ip_origin_is_x_forwarded = args.captcha_ip_origin_is_x_forwarded + self.captcha_ip_origin_is_x_forwarded = ( + args.captcha_ip_origin_is_x_forwarded + ) @classmethod def add_arguments(cls, parser): @@ -32,11 +35,12 @@ class CaptchaConfig(Config): ) group.add_argument( "--enable-registration-captcha", type=bool, default=False, - help="Enables ReCaptcha checks when registering, preventing signup "+ - "unless a captcha is answered. Requires a valid ReCaptcha public/private key." + help="Enables ReCaptcha checks when registering, preventing signup" + + " unless a captcha is answered. Requires a valid ReCaptcha " + + "public/private key." ) group.add_argument( "--captcha_ip_origin_is_x_forwarded", type=bool, default=False, - help="When checking captchas, use the X-Forwarded-For (XFF) header as the client IP "+ - "and not the actual client IP." + help="When checking captchas, use the X-Forwarded-For (XFF) header" + + " as the client IP and not the actual client IP." ) \ No newline at end of file -- cgit 1.4.1 From 28bcd01e8db14a70e020cd7dd188981f71e70258 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 19 Sep 2014 14:45:21 +0100 Subject: SYN-47: Fix bug where we still returned events for rooms we had left. SYN-47 #resolve --- synapse/storage/stream.py | 2 +- tests/storage/test_stream.py | 173 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 tests/storage/test_stream.py (limited to 'synapse') diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8c766b8a00..a76fecf24f 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -146,7 +146,7 @@ class StreamStore(SQLBaseStore): current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " "INNER JOIN current_state_events as c ON m.event_id = c.event_id " - "WHERE m.user_id = ?" + "WHERE m.user_id = ? AND m.membership = 'join'" ) # We also want to get any membership events about that user, e.g. diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py new file mode 100644 index 0000000000..6cbfcc7652 --- /dev/null +++ b/tests/storage/test_stream.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest +from twisted.internet import defer + +from synapse.server import HomeServer +from synapse.api.constants import Membership +from synapse.api.events.room import RoomMemberEvent, MessageEvent + +from tests.utils import SQLiteMemoryDbPool + + +class StreamStoreTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "test", + db_pool=db_pool, + ) + + self.store = hs.get_datastore() + self.event_factory = hs.get_event_factory() + + self.u_alice = hs.parse_userid("@alice:test") + self.u_bob = hs.parse_userid("@bob:test") + + self.room1 = hs.parse_roomid("!abc123:test") + self.room2 = hs.parse_roomid("!xyx987:test") + + self.depth = 1 + + @defer.inlineCallbacks + def inject_room_member(self, room, user, membership): + self.depth += 1 + + # Have to create a join event using the eventfactory + yield self.store.persist_event( + self.event_factory.create_event( + etype=RoomMemberEvent.TYPE, + user_id=user.to_string(), + state_key=user.to_string(), + room_id=room.to_string(), + membership=membership, + content={"membership": membership}, + depth=self.depth, + ) + ) + + + @defer.inlineCallbacks + def inject_message(self, room, user, body): + self.depth += 1 + + # Have to create a join event using the eventfactory + yield self.store.persist_event( + self.event_factory.create_event( + etype=MessageEvent.TYPE, + user_id=user.to_string(), + room_id=room.to_string(), + content={"body": body, "msgtype": u"message"}, + depth=self.depth, + ) + ) + + @defer.inlineCallbacks + def test_event_stream_get_other(self): + # Both bob and alice joins the room + yield self.inject_room_member(self.room1, self.u_alice, Membership.JOIN) + yield self.inject_room_member(self.room1, self.u_bob, Membership.JOIN) + + # Initial stream key: + start = yield self.store.get_room_events_max_id() + + yield self.inject_message(self.room1, self.u_alice, u"test") + + end = yield self.store.get_room_events_max_id() + + results, _ = yield self.store.get_room_events_stream( + self.u_bob.to_string(), + start, + end, + None, # Is currently ignored + ) + + self.assertEqual(1, len(results)) + + event = results[0] + + self.assertObjectHasAttributes( + { + "type": MessageEvent.TYPE, + "user_id": self.u_alice.to_string(), + "content": {"body": "test", "msgtype": "message"}, + }, + event, + ) + + @defer.inlineCallbacks + def test_event_stream_get_own(self): + # Both bob and alice joins the room + yield self.inject_room_member(self.room1, self.u_alice, Membership.JOIN) + yield self.inject_room_member(self.room1, self.u_bob, Membership.JOIN) + + # Initial stream key: + start = yield self.store.get_room_events_max_id() + + yield self.inject_message(self.room1, self.u_alice, u"test") + + end = yield self.store.get_room_events_max_id() + + results, _ = yield self.store.get_room_events_stream( + self.u_alice.to_string(), + start, + end, + None, # Is currently ignored + ) + + self.assertEqual(1, len(results)) + + event = results[0] + + self.assertObjectHasAttributes( + { + "type": MessageEvent.TYPE, + "user_id": self.u_alice.to_string(), + "content": {"body": "test", "msgtype": "message"}, + }, + event, + ) + + @defer.inlineCallbacks + def test_event_stream_join_leave(self): + # Both bob and alice joins the room + yield self.inject_room_member(self.room1, self.u_alice, Membership.JOIN) + yield self.inject_room_member(self.room1, self.u_bob, Membership.JOIN) + + # Then bob leaves again. + yield self.inject_room_member(self.room1, self.u_bob, Membership.LEAVE) + + # Initial stream key: + start = yield self.store.get_room_events_max_id() + + yield self.inject_message(self.room1, self.u_alice, u"test") + + end = yield self.store.get_room_events_max_id() + + results, _ = yield self.store.get_room_events_stream( + self.u_bob.to_string(), + start, + end, + None, # Is currently ignored + ) + + # We should not get the message, as it happened *after* bob left. + self.assertEqual(0, len(results)) -- cgit 1.4.1 From 176e3fd141ba9485ead8658e90ba000cef8db02c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 22 Sep 2014 17:42:09 +0100 Subject: Bump versions and changelog --- CHANGES.rst | 20 ++++++++++++++++++++ VERSION | 2 +- synapse/__init__.py | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/CHANGES.rst b/CHANGES.rst index dd188cf86a..400ded0f15 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,23 @@ +Changes in synapse 0.3.3 (2014-09-22) +===================================== + +Homeserver: + * Fix bug where you continued to get events for rooms you had left. + +Webclient: + * Add support for video calls with basic UI. + * Fix bug where one to one chats were named after your display name rather + than the other person's. + * Fix bug which caused lag when typing in the textarea. + * Refuse to run on browsers we know won't work. + * Trigger pagination when joining new rooms. + * Fix bug where we sometimes didn't display invitations in recents. + * Automatically join room when accepting a VoIP call. + * Disable outgoing and reject incoming calls on browsers we don't support + VoIP in. + * Don't display desktop notifications for messages in the room you are + non-idle and speaking in. + Changes in synapse 0.3.2 (2014-09-18) ===================================== diff --git a/VERSION b/VERSION index d15723fbe8..1c09c74e22 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.2 +0.3.3 diff --git a/synapse/__init__.py b/synapse/__init__.py index 0dbad9a0de..bba551b2c4 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.3.2" +__version__ = "0.3.3" -- cgit 1.4.1