From 2d3462714e48dca46dd54b17ca29188a17261e28 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 18 Aug 2015 14:22:02 +0100 Subject: Issue macaroons as opaque auth tokens This just replaces random bytes with macaroons. The macaroons are not inspected by the client or server. In particular, they claim to have an expiry time, but nothing verifies that they have not expired. Follow-up commits will actually enforce the expiration, and allow for token refresh. See https://bit.ly/matrix-auth for more information --- synapse/auth/macaroons.py | 0 synapse/config/registration.py | 4 +++ synapse/handlers/register.py | 19 +++++++---- synapse/python_dependencies.py | 1 + tests/handlers/test_register.py | 70 +++++++++++++++++++++++++++++++++++++++++ tests/utils.py | 2 ++ 6 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 synapse/auth/macaroons.py create mode 100644 tests/handlers/test_register.py diff --git a/synapse/auth/macaroons.py b/synapse/auth/macaroons.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 67e780864e..62de4b399f 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -32,9 +32,11 @@ class RegistrationConfig(Config): ) self.registration_shared_secret = config.get("registration_shared_secret") + self.macaroon_secret_key = config.get("macaroon_secret_key") def default_config(self, config_dir, server_name): registration_shared_secret = random_string_with_symbols(50) + macaroon_secret_key = random_string_with_symbols(50) return """\ ## Registration ## @@ -44,6 +46,8 @@ class RegistrationConfig(Config): # If set, allows registration by anyone who also has the shared # secret, even if registration is otherwise disabled. registration_shared_secret: "%(registration_shared_secret)s" + + macaroon_secret_key: "%(macaroon_secret_key)s" """ % locals() def add_arguments(self, parser): diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 39392d9fdd..86bacdda1d 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -25,9 +25,9 @@ import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor from synapse.http.client import CaptchaServerHttpClient -import base64 import bcrypt import logging +import pymacaroons import urllib logger = logging.getLogger(__name__) @@ -274,11 +274,18 @@ class RegistrationHandler(BaseHandler): ) def generate_token(self, user_id): - # urlsafe variant uses _ and - so use . as the separator and replace - # all =s with .s so http clients don't quote =s when it is used as - # query params. - return (base64.urlsafe_b64encode(user_id).replace('=', '.') + '.' + - stringutils.random_string(18)) + macaroon = pymacaroons.Macaroon( + location = self.hs.config.server_name, + identifier = "key", + key = self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("user_id = %s" % user_id) + macaroon.add_first_party_caveat("type = access") + now = self.hs.get_clock().time() + expiry = now + 60 * 60 + macaroon.add_first_party_caveat("time < %s" % expiry) + + return macaroon.serialize() def _generate_user_id(self): return "-" + stringutils.random_string(18) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 115bee8c41..b6e00c27b5 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -33,6 +33,7 @@ REQUIREMENTS = { "ujson": ["ujson"], "blist": ["blist"], "pysaml2": ["saml2"], + "pymacaroons": ["pymacaroons"], } CONDITIONAL_REQUIREMENTS = { "web_client": { diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py new file mode 100644 index 0000000000..b28b1a7ef0 --- /dev/null +++ b/tests/handlers/test_register.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pymacaroons + +from mock import Mock, NonCallableMock +from synapse.handlers.register import RegistrationHandler +from tests import unittest +from tests.utils import setup_test_homeserver +from twisted.internet import defer + + +class RegisterHandlers(object): + def __init__(self, hs): + self.registration_handler = RegistrationHandler(hs) + + +class RegisterTestCase(unittest.TestCase): + @defer.inlineCallbacks + def setUp(self): + self.hs = yield setup_test_homeserver(handlers=None) + self.hs.handlers = RegisterHandlers(self.hs) + + def test_token_is_a_macaroon(self): + self.hs.config.macaroon_secret_key = "this key is a huge secret" + + token = self.hs.handlers.registration_handler.generate_token("some_user") + # Check that we can parse the thing with pymacaroons + macaroon = pymacaroons.Macaroon.deserialize(token) + # The most basic of sanity checks + if "some_user" not in macaroon.inspect(): + self.fail("some_user was not in %s" % macaroon.inspect()) + + def test_macaroon_caveats(self): + self.hs.config.macaroon_secret_key = "this key is a massive secret" + self.hs.clock.now = 5000 + + token = self.hs.handlers.registration_handler.generate_token("a_user") + macaroon = pymacaroons.Macaroon.deserialize(token) + + def verify_gen(caveat): + return caveat == "gen = 1" + + def verify_user(caveat): + return caveat == "user_id = a_user" + + def verify_type(caveat): + return caveat == "type = access" + + def verify_expiry(caveat): + return caveat == "time < 8600" + + v = pymacaroons.Verifier() + v.satisfy_general(verify_gen) + v.satisfy_general(verify_user) + v.satisfy_general(verify_type) + v.satisfy_general(verify_expiry) + v.verify(macaroon, self.hs.config.macaroon_secret_key) \ No newline at end of file diff --git a/tests/utils.py b/tests/utils.py index eb035cf48f..80be70b74f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -44,6 +44,8 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.signing_key = [MockKey()] config.event_cache_size = 1 config.disable_registration = False + config.macaroon_secret_key = "not even a little secret" + config.server_name = "server.under.test" if "clock" not in kargs: kargs["clock"] = MockClock() -- cgit 1.5.1 From cacdb529abcfeefc4b4332db00dbf5eb6f50f016 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 18 Aug 2015 14:27:23 +0100 Subject: Remove accidentally added file --- synapse/auth/macaroons.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 synapse/auth/macaroons.py diff --git a/synapse/auth/macaroons.py b/synapse/auth/macaroons.py deleted file mode 100644 index e69de29bb2..0000000000 -- cgit 1.5.1 From 3e6fdfda002de6971b74aba7805ebdeb2b1b426d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 18 Aug 2015 15:18:50 +0100 Subject: Fix some formatting to use tuples --- synapse/handlers/register.py | 8 ++++---- tests/handlers/test_register.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 86bacdda1d..c391c1bdf5 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -279,11 +279,11 @@ class RegistrationHandler(BaseHandler): identifier = "key", key = self.hs.config.macaroon_secret_key) macaroon.add_first_party_caveat("gen = 1") - macaroon.add_first_party_caveat("user_id = %s" % user_id) + macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) macaroon.add_first_party_caveat("type = access") - now = self.hs.get_clock().time() - expiry = now + 60 * 60 - macaroon.add_first_party_caveat("time < %s" % expiry) + now = self.hs.get_clock().time_msec() + expiry = now + (60 * 60 * 1000) + macaroon.add_first_party_caveat("time < %d" % (expiry,)) return macaroon.serialize() diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index b28b1a7ef0..0766affe81 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -67,4 +67,4 @@ class RegisterTestCase(unittest.TestCase): v.satisfy_general(verify_user) v.satisfy_general(verify_type) v.satisfy_general(verify_expiry) - v.verify(macaroon, self.hs.config.macaroon_secret_key) \ No newline at end of file + v.verify(macaroon, self.hs.config.macaroon_secret_key) -- cgit 1.5.1 From 42e858daeb59b86c451e3f49d40c1f418c8f0748 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 18 Aug 2015 17:38:37 +0100 Subject: Fix units in test I made the non-test seconds instead of ms, but not the test --- tests/handlers/test_register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 0766affe81..91cc90242f 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -60,7 +60,7 @@ class RegisterTestCase(unittest.TestCase): return caveat == "type = access" def verify_expiry(caveat): - return caveat == "time < 8600" + return caveat == "time < 8600000" v = pymacaroons.Verifier() v.satisfy_general(verify_gen) -- cgit 1.5.1 From ce832c38d4ba1412cd5b5f8a4fb9328cb2d444fa Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 18 Aug 2015 17:39:26 +0100 Subject: Remove padding space around caveat operators --- synapse/handlers/register.py | 8 ++++---- tests/handlers/test_register.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c391c1bdf5..557aec4e6c 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -278,12 +278,12 @@ class RegistrationHandler(BaseHandler): location = self.hs.config.server_name, identifier = "key", key = self.hs.config.macaroon_secret_key) - macaroon.add_first_party_caveat("gen = 1") - macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) - macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("gen=1") + macaroon.add_first_party_caveat("user_id=%s" % (user_id,)) + macaroon.add_first_party_caveat("type=access") now = self.hs.get_clock().time_msec() expiry = now + (60 * 60 * 1000) - macaroon.add_first_party_caveat("time < %d" % (expiry,)) + macaroon.add_first_party_caveat("time<%d" % (expiry,)) return macaroon.serialize() diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 91cc90242f..18507c547d 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -51,16 +51,16 @@ class RegisterTestCase(unittest.TestCase): macaroon = pymacaroons.Macaroon.deserialize(token) def verify_gen(caveat): - return caveat == "gen = 1" + return caveat == "gen=1" def verify_user(caveat): - return caveat == "user_id = a_user" + return caveat == "user_id=a_user" def verify_type(caveat): - return caveat == "type = access" + return caveat == "type=access" def verify_expiry(caveat): - return caveat == "time < 8600000" + return caveat == "time<8600000" v = pymacaroons.Verifier() v.satisfy_general(verify_gen) -- cgit 1.5.1 From 7f08ebb7729fdfac2b5e957692e89f97e70c9a06 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 19 Aug 2015 13:21:36 +0100 Subject: Switch to pymacaroons-pynacl --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 94d7784aee..fa24199377 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -33,7 +33,7 @@ REQUIREMENTS = { "ujson": ["ujson"], "blist": ["blist"], "pysaml2": ["saml2"], - "pymacaroons": ["pymacaroons"], + "pymacaroons-pynacl": ["pymacaroons"], } CONDITIONAL_REQUIREMENTS = { "web_client": { -- cgit 1.5.1 From 70e265e695a67a412b5ac76cc9bae71e9d384e80 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 19 Aug 2015 14:30:31 +0100 Subject: Re-add whitespace around caveat operators --- synapse/handlers/register.py | 8 ++++---- tests/handlers/test_register.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 557aec4e6c..c391c1bdf5 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -278,12 +278,12 @@ class RegistrationHandler(BaseHandler): location = self.hs.config.server_name, identifier = "key", key = self.hs.config.macaroon_secret_key) - macaroon.add_first_party_caveat("gen=1") - macaroon.add_first_party_caveat("user_id=%s" % (user_id,)) - macaroon.add_first_party_caveat("type=access") + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) + macaroon.add_first_party_caveat("type = access") now = self.hs.get_clock().time_msec() expiry = now + (60 * 60 * 1000) - macaroon.add_first_party_caveat("time<%d" % (expiry,)) + macaroon.add_first_party_caveat("time < %d" % (expiry,)) return macaroon.serialize() diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 18507c547d..91cc90242f 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -51,16 +51,16 @@ class RegisterTestCase(unittest.TestCase): macaroon = pymacaroons.Macaroon.deserialize(token) def verify_gen(caveat): - return caveat == "gen=1" + return caveat == "gen = 1" def verify_user(caveat): - return caveat == "user_id=a_user" + return caveat == "user_id = a_user" def verify_type(caveat): - return caveat == "type=access" + return caveat == "type = access" def verify_expiry(caveat): - return caveat == "time<8600000" + return caveat == "time < 8600000" v = pymacaroons.Verifier() v.satisfy_general(verify_gen) -- cgit 1.5.1 From 617501dd2a0562f4bf7edf8bc7a4e8aeb16b2254 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 20 Aug 2015 11:35:56 +0100 Subject: Move token generation to auth handler I prefer the auth handler to worry about all auth, and register to call into it as needed, than to smatter auth logic between the two. --- synapse/handlers/auth.py | 29 ++++++++++++++--- synapse/handlers/register.py | 26 +++++---------- tests/handlers/test_auth.py | 70 +++++++++++++++++++++++++++++++++++++++++ tests/handlers/test_register.py | 70 ----------------------------------------- 4 files changed, 101 insertions(+), 94 deletions(-) create mode 100644 tests/handlers/test_auth.py delete mode 100644 tests/handlers/test_register.py diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index be2baeaece..0bf917efdd 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -26,6 +26,7 @@ from twisted.web.client import PartialDownloadError import logging import bcrypt +import pymacaroons import simplejson import synapse.util.stringutils as stringutils @@ -284,12 +285,9 @@ class AuthHandler(BaseHandler): LoginError if there was an authentication problem. """ yield self._check_password(user_id, password) - - reg_handler = self.hs.get_handlers().registration_handler - access_token = reg_handler.generate_token(user_id) logger.info("Logging in user %s", user_id) - yield self.store.add_access_token_to_user(user_id, access_token) - defer.returnValue(access_token) + token = yield self.issue_access_token(user_id) + defer.returnValue(token) @defer.inlineCallbacks def _check_password(self, user_id, password): @@ -304,6 +302,27 @@ class AuthHandler(BaseHandler): logger.warn("Failed password login for user %s", user_id) raise LoginError(403, "", errcode=Codes.FORBIDDEN) + @defer.inlineCallbacks + def issue_access_token(self, user_id): + reg_handler = self.hs.get_handlers().registration_handler + access_token = reg_handler.generate_access_token(user_id) + yield self.store.add_access_token_to_user(user_id, access_token) + defer.returnValue(access_token) + + def generate_access_token(self, user_id): + macaroon = pymacaroons.Macaroon( + location = self.hs.config.server_name, + identifier = "key", + key = self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) + macaroon.add_first_party_caveat("type = access") + now = self.hs.get_clock().time_msec() + expiry = now + (60 * 60 * 1000) + macaroon.add_first_party_caveat("time < %d" % (expiry,)) + + return macaroon.serialize() + @defer.inlineCallbacks def set_password(self, user_id, newpassword): password_hash = bcrypt.hashpw(newpassword, bcrypt.gensalt()) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c391c1bdf5..3d1b6531c2 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -27,7 +27,6 @@ from synapse.http.client import CaptchaServerHttpClient import bcrypt import logging -import pymacaroons import urllib logger = logging.getLogger(__name__) @@ -91,7 +90,7 @@ class RegistrationHandler(BaseHandler): user = UserID(localpart, self.hs.hostname) user_id = user.to_string() - token = self.generate_token(user_id) + token = self.auth_handler().generate_access_token(user_id) yield self.store.register( user_id=user_id, token=token, @@ -111,7 +110,7 @@ class RegistrationHandler(BaseHandler): user_id = user.to_string() yield self.check_user_id_is_valid(user_id) - token = self.generate_token(user_id) + token = self.auth_handler().generate_access_token(user_id) yield self.store.register( user_id=user_id, token=token, @@ -161,7 +160,7 @@ class RegistrationHandler(BaseHandler): 400, "Invalid user localpart for this application service.", errcode=Codes.EXCLUSIVE ) - token = self.generate_token(user_id) + token = self.auth_handler().generate_access_token(user_id) yield self.store.register( user_id=user_id, token=token, @@ -208,7 +207,7 @@ class RegistrationHandler(BaseHandler): user_id = user.to_string() yield self.check_user_id_is_valid(user_id) - token = self.generate_token(user_id) + token = self.auth_handler().generate_access_token(user_id) try: yield self.store.register( user_id=user_id, @@ -273,20 +272,6 @@ class RegistrationHandler(BaseHandler): errcode=Codes.EXCLUSIVE ) - def generate_token(self, user_id): - macaroon = pymacaroons.Macaroon( - location = self.hs.config.server_name, - identifier = "key", - key = self.hs.config.macaroon_secret_key) - macaroon.add_first_party_caveat("gen = 1") - macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) - macaroon.add_first_party_caveat("type = access") - now = self.hs.get_clock().time_msec() - expiry = now + (60 * 60 * 1000) - macaroon.add_first_party_caveat("time < %d" % (expiry,)) - - return macaroon.serialize() - def _generate_user_id(self): return "-" + stringutils.random_string(18) @@ -329,3 +314,6 @@ class RegistrationHandler(BaseHandler): } ) defer.returnValue(data) + + def auth_handler(self): + return self.hs.get_handlers().auth_handler diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py new file mode 100644 index 0000000000..978e4d0d2e --- /dev/null +++ b/tests/handlers/test_auth.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pymacaroons + +from mock import Mock, NonCallableMock +from synapse.handlers.auth import AuthHandler +from tests import unittest +from tests.utils import setup_test_homeserver +from twisted.internet import defer + + +class AuthHandlers(object): + def __init__(self, hs): + self.auth_handler = AuthHandler(hs) + + +class AuthTestCase(unittest.TestCase): + @defer.inlineCallbacks + def setUp(self): + self.hs = yield setup_test_homeserver(handlers=None) + self.hs.handlers = AuthHandlers(self.hs) + + def test_token_is_a_macaroon(self): + self.hs.config.macaroon_secret_key = "this key is a huge secret" + + token = self.hs.handlers.auth_handler.generate_access_token("some_user") + # Check that we can parse the thing with pymacaroons + macaroon = pymacaroons.Macaroon.deserialize(token) + # The most basic of sanity checks + if "some_user" not in macaroon.inspect(): + self.fail("some_user was not in %s" % macaroon.inspect()) + + def test_macaroon_caveats(self): + self.hs.config.macaroon_secret_key = "this key is a massive secret" + self.hs.clock.now = 5000 + + token = self.hs.handlers.auth_handler.generate_access_token("a_user") + macaroon = pymacaroons.Macaroon.deserialize(token) + + def verify_gen(caveat): + return caveat == "gen = 1" + + def verify_user(caveat): + return caveat == "user_id = a_user" + + def verify_type(caveat): + return caveat == "type = access" + + def verify_expiry(caveat): + return caveat == "time < 8600000" + + v = pymacaroons.Verifier() + v.satisfy_general(verify_gen) + v.satisfy_general(verify_user) + v.satisfy_general(verify_type) + v.satisfy_general(verify_expiry) + v.verify(macaroon, self.hs.config.macaroon_secret_key) diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py deleted file mode 100644 index 91cc90242f..0000000000 --- a/tests/handlers/test_register.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pymacaroons - -from mock import Mock, NonCallableMock -from synapse.handlers.register import RegistrationHandler -from tests import unittest -from tests.utils import setup_test_homeserver -from twisted.internet import defer - - -class RegisterHandlers(object): - def __init__(self, hs): - self.registration_handler = RegistrationHandler(hs) - - -class RegisterTestCase(unittest.TestCase): - @defer.inlineCallbacks - def setUp(self): - self.hs = yield setup_test_homeserver(handlers=None) - self.hs.handlers = RegisterHandlers(self.hs) - - def test_token_is_a_macaroon(self): - self.hs.config.macaroon_secret_key = "this key is a huge secret" - - token = self.hs.handlers.registration_handler.generate_token("some_user") - # Check that we can parse the thing with pymacaroons - macaroon = pymacaroons.Macaroon.deserialize(token) - # The most basic of sanity checks - if "some_user" not in macaroon.inspect(): - self.fail("some_user was not in %s" % macaroon.inspect()) - - def test_macaroon_caveats(self): - self.hs.config.macaroon_secret_key = "this key is a massive secret" - self.hs.clock.now = 5000 - - token = self.hs.handlers.registration_handler.generate_token("a_user") - macaroon = pymacaroons.Macaroon.deserialize(token) - - def verify_gen(caveat): - return caveat == "gen = 1" - - def verify_user(caveat): - return caveat == "user_id = a_user" - - def verify_type(caveat): - return caveat == "type = access" - - def verify_expiry(caveat): - return caveat == "time < 8600000" - - v = pymacaroons.Verifier() - v.satisfy_general(verify_gen) - v.satisfy_general(verify_user) - v.satisfy_general(verify_type) - v.satisfy_general(verify_expiry) - v.verify(macaroon, self.hs.config.macaroon_secret_key) -- cgit 1.5.1 From 13a6517d89c0619a938321640f331571eba0edc9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 20 Aug 2015 16:01:29 +0100 Subject: s/by_token/by_access_token/g We're about to have two kinds of token, access and refresh --- synapse/api/auth.py | 6 +++--- synapse/storage/registration.py | 6 +++--- tests/api/test_auth.py | 16 ++++++++-------- tests/rest/client/v1/test_presence.py | 8 ++++---- tests/rest/client/v1/test_rooms.py | 28 ++++++++++++++-------------- tests/rest/client/v1/test_typing.py | 4 ++-- tests/rest/client/v1/utils.py | 2 +- tests/rest/client/v2_alpha/__init__.py | 4 ++-- tests/storage/test_registration.py | 4 ++-- tests/utils.py | 2 +- 10 files changed, 40 insertions(+), 40 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 1e3b0fbfb7..3d9237ccc3 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -361,7 +361,7 @@ class Auth(object): except KeyError: pass # normal users won't have the user_id query parameter set. - user_info = yield self.get_user_by_token(access_token) + user_info = yield self.get_user_by_access_token(access_token) user = user_info["user"] device_id = user_info["device_id"] token_id = user_info["token_id"] @@ -390,7 +390,7 @@ class Auth(object): ) @defer.inlineCallbacks - def get_user_by_token(self, token): + def get_user_by_access_token(self, token): """ Get a registered user's ID. Args: @@ -401,7 +401,7 @@ class Auth(object): Raises: AuthError if no user by that token exists or the token is invalid. """ - ret = yield self.store.get_user_by_token(token) + ret = yield self.store.get_user_by_access_token(token) if not ret: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Unrecognised access token.", diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index bf803f2c6e..0e404afb7c 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -132,10 +132,10 @@ class RegistrationStore(SQLBaseStore): user_id ) for r in rows: - self.get_user_by_token.invalidate((r,)) + self.get_user_by_access_token.invalidate((r,)) @cached() - def get_user_by_token(self, token): + def get_user_by_access_token(self, token): """Get a user from the given access token. Args: @@ -147,7 +147,7 @@ class RegistrationStore(SQLBaseStore): StoreError if no user was found. """ return self.runInteraction( - "get_user_by_token", + "get_user_by_access_token", self._query_for_auth, token ) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 4f83db5e84..3343c635cc 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -44,7 +44,7 @@ class AuthTestCase(unittest.TestCase): "token_id": "ditto", "admin": False } - self.store.get_user_by_token = Mock(return_value=user_info) + self.store.get_user_by_access_token = Mock(return_value=user_info) request = Mock(args={}) request.args["access_token"] = [self.test_token] @@ -54,7 +54,7 @@ class AuthTestCase(unittest.TestCase): def test_get_user_by_req_user_bad_token(self): self.store.get_app_service_by_token = Mock(return_value=None) - self.store.get_user_by_token = Mock(return_value=None) + self.store.get_user_by_access_token = Mock(return_value=None) request = Mock(args={}) request.args["access_token"] = [self.test_token] @@ -70,7 +70,7 @@ class AuthTestCase(unittest.TestCase): "token_id": "ditto", "admin": False } - self.store.get_user_by_token = Mock(return_value=user_info) + self.store.get_user_by_access_token = Mock(return_value=user_info) request = Mock(args={}) request.requestHeaders.getRawHeaders = Mock(return_value=[""]) @@ -81,7 +81,7 @@ class AuthTestCase(unittest.TestCase): def test_get_user_by_req_appservice_valid_token(self): app_service = Mock(token="foobar", url="a_url", sender=self.test_user) self.store.get_app_service_by_token = Mock(return_value=app_service) - self.store.get_user_by_token = Mock(return_value=None) + self.store.get_user_by_access_token = Mock(return_value=None) request = Mock(args={}) request.args["access_token"] = [self.test_token] @@ -91,7 +91,7 @@ class AuthTestCase(unittest.TestCase): def test_get_user_by_req_appservice_bad_token(self): self.store.get_app_service_by_token = Mock(return_value=None) - self.store.get_user_by_token = Mock(return_value=None) + self.store.get_user_by_access_token = Mock(return_value=None) request = Mock(args={}) request.args["access_token"] = [self.test_token] @@ -102,7 +102,7 @@ class AuthTestCase(unittest.TestCase): def test_get_user_by_req_appservice_missing_token(self): app_service = Mock(token="foobar", url="a_url", sender=self.test_user) self.store.get_app_service_by_token = Mock(return_value=app_service) - self.store.get_user_by_token = Mock(return_value=None) + self.store.get_user_by_access_token = Mock(return_value=None) request = Mock(args={}) request.requestHeaders.getRawHeaders = Mock(return_value=[""]) @@ -115,7 +115,7 @@ class AuthTestCase(unittest.TestCase): app_service = Mock(token="foobar", url="a_url", sender=self.test_user) app_service.is_interested_in_user = Mock(return_value=True) self.store.get_app_service_by_token = Mock(return_value=app_service) - self.store.get_user_by_token = Mock(return_value=None) + self.store.get_user_by_access_token = Mock(return_value=None) request = Mock(args={}) request.args["access_token"] = [self.test_token] @@ -129,7 +129,7 @@ class AuthTestCase(unittest.TestCase): app_service = Mock(token="foobar", url="a_url", sender=self.test_user) app_service.is_interested_in_user = Mock(return_value=False) self.store.get_app_service_by_token = Mock(return_value=app_service) - self.store.get_user_by_token = Mock(return_value=None) + self.store.get_user_by_access_token = Mock(return_value=None) request = Mock(args={}) request.args["access_token"] = [self.test_token] diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 089a71568c..0b78a82a66 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -70,7 +70,7 @@ class PresenceStateTestCase(unittest.TestCase): return defer.succeed([]) self.datastore.get_presence_list = get_presence_list - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(myid), "admin": False, @@ -78,7 +78,7 @@ class PresenceStateTestCase(unittest.TestCase): "token_id": 1, } - hs.get_v1auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token room_member_handler = hs.handlers.room_member_handler = Mock( spec=[ @@ -159,7 +159,7 @@ class PresenceListTestCase(unittest.TestCase): ) self.datastore.has_presence_state = has_presence_state - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(myid), "admin": False, @@ -173,7 +173,7 @@ class PresenceListTestCase(unittest.TestCase): ] ) - hs.get_v1auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token presence.register_servlets(hs, self.mock_resource) diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index c83348acf9..2e55cc08a1 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -54,14 +54,14 @@ class RoomPermissionsTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, "token_id": 1, } - hs.get_v1auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -441,14 +441,14 @@ class RoomsMemberListTestCase(RestTestCase): self.auth_user_id = self.user_id - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, "token_id": 1, } - hs.get_v1auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -521,14 +521,14 @@ class RoomsCreateTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, "token_id": 1, } - hs.get_v1auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -614,7 +614,7 @@ class RoomTopicTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), "admin": False, @@ -622,7 +622,7 @@ class RoomTopicTestCase(RestTestCase): "token_id": 1, } - hs.get_v1auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -721,14 +721,14 @@ class RoomMemberStateTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, "token_id": 1, } - hs.get_v1auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -848,14 +848,14 @@ class RoomMessagesTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, "token_id": 1, } - hs.get_v1auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -945,14 +945,14 @@ class RoomInitialSyncTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, "token_id": 1, } - hs.get_v1auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 7d8b1c2683..dc8bbaaf0e 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -61,7 +61,7 @@ class RoomTypingTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), "admin": False, @@ -69,7 +69,7 @@ class RoomTypingTestCase(RestTestCase): "token_id": 1, } - hs.get_v1auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index 579441fb4a..c472d53043 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -37,7 +37,7 @@ class RestTestCase(unittest.TestCase): self.mock_resource = None self.auth_user_id = None - def mock_get_user_by_token(self, token=None): + def mock_get_user_by_access_token(self, token=None): return self.auth_user_id @defer.inlineCallbacks diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index de5a917e6a..15568b36cd 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -43,14 +43,14 @@ class V2AlphaRestTestCase(unittest.TestCase): resource_for_federation=self.mock_resource, ) - def _get_user_by_token(token=None): + def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.USER_ID), "admin": False, "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_auth().get_user_by_access_token = _get_user_by_access_token for r in self.TO_REGISTER: r.register_servlets(hs, self.mock_resource) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 2702291178..7a24cf898a 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -46,7 +46,7 @@ class RegistrationStoreTestCase(unittest.TestCase): (yield self.store.get_user_by_id(self.user_id)) ) - result = yield self.store.get_user_by_token(self.tokens[0]) + result = yield self.store.get_user_by_access_token(self.tokens[0]) self.assertDictContainsSubset( { @@ -64,7 +64,7 @@ class RegistrationStoreTestCase(unittest.TestCase): yield self.store.register(self.user_id, self.tokens[0], self.pwhash) yield self.store.add_access_token_to_user(self.user_id, self.tokens[1]) - result = yield self.store.get_user_by_token(self.tokens[1]) + result = yield self.store.get_user_by_access_token(self.tokens[1]) self.assertDictContainsSubset( { diff --git a/tests/utils.py b/tests/utils.py index 80be70b74f..d0fba2252d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -277,7 +277,7 @@ class MemoryDataStore(object): raise StoreError(400, "User in use.") self.tokens_to_users[token] = user_id - def get_user_by_token(self, token): + def get_user_by_access_token(self, token): try: return { "name": self.tokens_to_users[token], -- cgit 1.5.1 From cecbd636e94f4e900ef6d246b62698ff1c8ee352 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 20 Aug 2015 16:21:35 +0100 Subject: /tokenrefresh POST endpoint This allows refresh tokens to be exchanged for (access_token, refresh_token). It also starts issuing them on login, though no clients currently interpret them. --- synapse/handlers/auth.py | 35 ++++++++++-- synapse/rest/client/v1/login.py | 6 ++- synapse/rest/client/v2_alpha/__init__.py | 2 + synapse/rest/client/v2_alpha/tokenrefresh.py | 56 +++++++++++++++++++ synapse/storage/__init__.py | 2 +- synapse/storage/_base.py | 1 + synapse/storage/registration.py | 62 ++++++++++++++++++++++ synapse/storage/schema/delta/23/refresh_tokens.sql | 21 ++++++++ tests/storage/test_registration.py | 55 +++++++++++++++++++ 9 files changed, 232 insertions(+), 8 deletions(-) create mode 100644 synapse/rest/client/v2_alpha/tokenrefresh.py create mode 100644 synapse/storage/schema/delta/23/refresh_tokens.sql diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 0bf917efdd..65bd8189dc 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -279,15 +279,18 @@ class AuthHandler(BaseHandler): user_id (str): User ID password (str): Password Returns: - The access token for the user's session. + A tuple of: + The access token for the user's session. + The refresh token for the user's session. Raises: StoreError if there was a problem storing the token. LoginError if there was an authentication problem. """ yield self._check_password(user_id, password) logger.info("Logging in user %s", user_id) - token = yield self.issue_access_token(user_id) - defer.returnValue(token) + access_token = yield self.issue_access_token(user_id) + refresh_token = yield self.issue_refresh_token(user_id) + defer.returnValue((access_token, refresh_token)) @defer.inlineCallbacks def _check_password(self, user_id, password): @@ -304,11 +307,16 @@ class AuthHandler(BaseHandler): @defer.inlineCallbacks def issue_access_token(self, user_id): - reg_handler = self.hs.get_handlers().registration_handler - access_token = reg_handler.generate_access_token(user_id) + access_token = self.generate_access_token(user_id) yield self.store.add_access_token_to_user(user_id, access_token) defer.returnValue(access_token) + @defer.inlineCallbacks + def issue_refresh_token(self, user_id): + refresh_token = self.generate_refresh_token(user_id) + yield self.store.add_refresh_token_to_user(user_id, refresh_token) + defer.returnValue(refresh_token) + def generate_access_token(self, user_id): macaroon = pymacaroons.Macaroon( location = self.hs.config.server_name, @@ -323,6 +331,23 @@ class AuthHandler(BaseHandler): return macaroon.serialize() + def generate_refresh_token(self, user_id): + m = self._generate_base_macaroon(user_id) + m.add_first_party_caveat("type = refresh") + # Important to add a nonce, because otherwise every refresh token for a + # user will be the same. + m.add_first_party_caveat("nonce = %s" % stringutils.random_string_with_symbols(16)) + return m.serialize() + + def _generate_base_macaroon(self, user_id): + macaroon = pymacaroons.Macaroon( + location = self.hs.config.server_name, + identifier = "key", + key = self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) + return macaroon + @defer.inlineCallbacks def set_password(self, user_id, newpassword): password_hash = bcrypt.hashpw(newpassword, bcrypt.gensalt()) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 694072693d..b963a38618 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -78,13 +78,15 @@ class LoginRestServlet(ClientV1RestServlet): login_submission["user"] = UserID.create( login_submission["user"], self.hs.hostname).to_string() - token = yield self.handlers.auth_handler.login_with_password( + auth_handler = self.handlers.auth_handler + access_token, refresh_token = yield auth_handler.login_with_password( user_id=login_submission["user"], password=login_submission["password"]) result = { "user_id": login_submission["user"], # may have changed - "access_token": token, + "access_token": access_token, + "refresh_token": refresh_token, "home_server": self.hs.hostname, } diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 33f961e898..5831ff0e62 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -21,6 +21,7 @@ from . import ( auth, receipts, keys, + tokenrefresh, ) from synapse.http.server import JsonResource @@ -42,3 +43,4 @@ class ClientV2AlphaRestResource(JsonResource): auth.register_servlets(hs, client_resource) receipts.register_servlets(hs, client_resource) keys.register_servlets(hs, client_resource) + tokenrefresh.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/tokenrefresh.py b/synapse/rest/client/v2_alpha/tokenrefresh.py new file mode 100644 index 0000000000..901e777983 --- /dev/null +++ b/synapse/rest/client/v2_alpha/tokenrefresh.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import AuthError, StoreError, SynapseError +from synapse.http.servlet import RestServlet + +from ._base import client_v2_pattern, parse_json_dict_from_request + + +class TokenRefreshRestServlet(RestServlet): + """ + Exchanges refresh tokens for a pair of an access token and a new refresh + token. + """ + PATTERN = client_v2_pattern("/tokenrefresh") + + def __init__(self, hs): + super(TokenRefreshRestServlet, self).__init__() + self.hs = hs + self.store = hs.get_datastore() + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_json_dict_from_request(request) + try: + old_refresh_token = body["refresh_token"] + auth_handler = self.hs.get_handlers().auth_handler + (user_id, new_refresh_token) = yield self.store.exchange_refresh_token( + old_refresh_token, auth_handler.generate_refresh_token) + new_access_token = yield auth_handler.issue_access_token(user_id) + defer.returnValue((200, { + "access_token": new_access_token, + "refresh_token": new_refresh_token, + })) + except KeyError: + raise SynapseError(400, "Missing required key 'refresh_token'.") + except StoreError: + raise AuthError(403, "Did not recognize refresh token") + + +def register_servlets(hs, http_server): + TokenRefreshRestServlet(hs).register(http_server) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f154b1c8ae..53673b3bf5 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 22 +SCHEMA_VERSION = 23 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1444767a52..ce71389f02 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -181,6 +181,7 @@ class SQLBaseStore(object): self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) self._state_groups_id_gen = IdGenerator("state_groups", "id", self) self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) + self._refresh_tokens_id_gen = IdGenerator("refresh_tokens", "id", self) self._pushers_id_gen = IdGenerator("pushers", "id", self) self._push_rule_id_gen = IdGenerator("push_rules", "id", self) self._push_rules_enable_id_gen = IdGenerator("push_rules_enable", "id", self) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 0e404afb7c..f632306688 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -50,6 +50,28 @@ class RegistrationStore(SQLBaseStore): desc="add_access_token_to_user", ) + @defer.inlineCallbacks + def add_refresh_token_to_user(self, user_id, token): + """Adds a refresh token for the given user. + + Args: + user_id (str): The user ID. + token (str): The new refresh token to add. + Raises: + StoreError if there was a problem adding this. + """ + next_id = yield self._refresh_tokens_id_gen.get_next() + + yield self._simple_insert( + "refresh_tokens", + { + "id": next_id, + "user_id": user_id, + "token": token + }, + desc="add_refresh_token_to_user", + ) + @defer.inlineCallbacks def register(self, user_id, token, password_hash): """Attempts to register an account. @@ -152,6 +174,46 @@ class RegistrationStore(SQLBaseStore): token ) + def exchange_refresh_token(self, refresh_token, token_generator): + """Exchange a refresh token for a new access token and refresh token. + + Doing so invalidates the old refresh token - refresh tokens are single + use. + + Args: + token (str): The refresh token of a user. + token_generator (fn: str -> str): Function which, when given a + user ID, returns a unique refresh token for that user. This + function must never return the same value twice. + Returns: + tuple of (user_id, refresh_token) + Raises: + StoreError if no user was found with that refresh token. + """ + return self.runInteraction( + "exchange_refresh_token", + self._exchange_refresh_token, + refresh_token, + token_generator + ) + + def _exchange_refresh_token(self, txn, old_token, token_generator): + sql = "SELECT user_id FROM refresh_tokens WHERE token = ?" + txn.execute(sql, (old_token,)) + rows = self.cursor_to_dict(txn) + if not rows: + raise StoreError(403, "Did not recognize refresh token") + user_id = rows[0]["user_id"] + + # TODO(danielwh): Maybe perform a validation on the macaroon that + # macaroon.user_id == user_id. + + new_token = token_generator(user_id) + sql = "UPDATE refresh_tokens SET token = ? WHERE token = ?" + txn.execute(sql, (new_token, old_token,)) + + return user_id, new_token + @defer.inlineCallbacks def is_server_admin(self, user): res = yield self._simple_select_one_onecol( diff --git a/synapse/storage/schema/delta/23/refresh_tokens.sql b/synapse/storage/schema/delta/23/refresh_tokens.sql new file mode 100644 index 0000000000..46839e016c --- /dev/null +++ b/synapse/storage/schema/delta/23/refresh_tokens.sql @@ -0,0 +1,21 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS refresh_tokens( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token TEXT NOT NULL, + user_id TEXT NOT NULL, + UNIQUE (token) +); diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 7a24cf898a..a4f929796a 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -17,7 +17,9 @@ from tests import unittest from twisted.internet import defer +from synapse.api.errors import StoreError from synapse.storage.registration import RegistrationStore +from synapse.util import stringutils from tests.utils import setup_test_homeserver @@ -27,6 +29,7 @@ class RegistrationStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): hs = yield setup_test_homeserver() + self.db_pool = hs.get_db_pool() self.store = RegistrationStore(hs) @@ -77,3 +80,55 @@ class RegistrationStoreTestCase(unittest.TestCase): self.assertTrue("token_id" in result) + @defer.inlineCallbacks + def test_exchange_refresh_token_valid(self): + uid = stringutils.random_string(32) + generator = TokenGenerator() + last_token = generator.generate(uid) + + self.db_pool.runQuery( + "INSERT INTO refresh_tokens(user_id, token) VALUES(?,?)", + (uid, last_token,)) + + (found_user_id, refresh_token) = yield self.store.exchange_refresh_token( + last_token, generator.generate) + self.assertEqual(uid, found_user_id) + + rows = yield self.db_pool.runQuery( + "SELECT token FROM refresh_tokens WHERE user_id = ?", (uid, )) + self.assertEqual([(refresh_token,)], rows) + # We issued token 1, then exchanged it for token 2 + expected_refresh_token = u"%s-%d" % (uid, 2,) + self.assertEqual(expected_refresh_token, refresh_token) + + @defer.inlineCallbacks + def test_exchange_refresh_token_none(self): + uid = stringutils.random_string(32) + generator = TokenGenerator() + last_token = generator.generate(uid) + + with self.assertRaises(StoreError): + yield self.store.exchange_refresh_token(last_token, generator.generate) + + @defer.inlineCallbacks + def test_exchange_refresh_token_invalid(self): + uid = stringutils.random_string(32) + generator = TokenGenerator() + last_token = generator.generate(uid) + wrong_token = "%s-wrong" % (last_token,) + + self.db_pool.runQuery( + "INSERT INTO refresh_tokens(user_id, token) VALUES(?,?)", + (uid, wrong_token,)) + + with self.assertRaises(StoreError): + yield self.store.exchange_refresh_token(last_token, generator.generate) + + +class TokenGenerator: + def __init__(self): + self._last_issued_token = 0 + + def generate(self, user_id): + self._last_issued_token += 1 + return u"%s-%d" % (user_id, self._last_issued_token,) -- cgit 1.5.1 From f90ea3dc73de6c2d0c1b7aa8a21fbf3a0feadcf9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 20 Aug 2015 16:42:17 +0100 Subject: Correct pynacl version to 0.3.0 0.0.3 was a typo --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index fa06480ad1..01fc4ea1e5 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -24,7 +24,7 @@ REQUIREMENTS = { "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], "pyasn1": ["pyasn1"], - "pynacl>=0.0.3": ["nacl>=0.0.3"], + "pynacl>=0.3.0": ["nacl>=0.3.0"], "daemonize": ["daemonize"], "py-bcrypt": ["bcrypt"], "frozendict>=0.4": ["frozendict"], -- cgit 1.5.1 From 225c244aba30ec7cc465395dc732785bc2969134 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 20 Aug 2015 17:10:10 +0100 Subject: Remove incorrect whitespace --- synapse/handlers/register.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c391c1bdf5..1adc3eebbb 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -275,9 +275,9 @@ class RegistrationHandler(BaseHandler): def generate_token(self, user_id): macaroon = pymacaroons.Macaroon( - location = self.hs.config.server_name, - identifier = "key", - key = self.hs.config.macaroon_secret_key) + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) macaroon.add_first_party_caveat("gen = 1") macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) macaroon.add_first_party_caveat("type = access") -- cgit 1.5.1 From ea570ffaebf59219c06d5d3d65400b1f1b1384bd Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 20 Aug 2015 17:22:41 +0100 Subject: Fix flake8 warnings --- synapse/handlers/auth.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 17465d2af6..1b0971e13d 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -336,14 +336,16 @@ class AuthHandler(BaseHandler): m.add_first_party_caveat("type = refresh") # Important to add a nonce, because otherwise every refresh token for a # user will be the same. - m.add_first_party_caveat("nonce = %s" % stringutils.random_string_with_symbols(16)) + m.add_first_party_caveat("nonce = %s" % ( + stringutils.random_string_with_symbols(16), + )) return m.serialize() def _generate_base_macaroon(self, user_id): macaroon = pymacaroons.Macaroon( - location = self.hs.config.server_name, - identifier = "key", - key = self.hs.config.macaroon_secret_key) + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) macaroon.add_first_party_caveat("gen = 1") macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) return macaroon -- cgit 1.5.1 From 8c74bd896010c6011a63bc7147e39e0078df7dcb Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 20 Aug 2015 17:26:52 +0100 Subject: Fix indentation --- synapse/handlers/auth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1b0971e13d..e043363176 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -321,7 +321,8 @@ class AuthHandler(BaseHandler): macaroon = pymacaroons.Macaroon( location=self.hs.config.server_name, identifier="key", - key=self.hs.config.macaroon_secret_key) + key=self.hs.config.macaroon_secret_key + ) macaroon.add_first_party_caveat("gen = 1") macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) macaroon.add_first_party_caveat("type = access") -- cgit 1.5.1 From c7788685b061dc1fbbecc07e472570f99f36dca3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 20 Aug 2015 17:43:12 +0100 Subject: Fix bad merge --- synapse/handlers/auth.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index e043363176..c983d444e8 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -318,18 +318,11 @@ class AuthHandler(BaseHandler): defer.returnValue(refresh_token) def generate_access_token(self, user_id): - macaroon = pymacaroons.Macaroon( - location=self.hs.config.server_name, - identifier="key", - key=self.hs.config.macaroon_secret_key - ) - macaroon.add_first_party_caveat("gen = 1") - macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) + macaroon = self._generate_base_macaroon(user_id) macaroon.add_first_party_caveat("type = access") now = self.hs.get_clock().time_msec() expiry = now + (60 * 60 * 1000) macaroon.add_first_party_caveat("time < %d" % (expiry,)) - return macaroon.serialize() def generate_refresh_token(self, user_id): -- cgit 1.5.1 From 21b71b6d7ce1aee5ce09f7c0f5d5d774dc61d60d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 20 Aug 2015 21:54:53 +0100 Subject: Return fully qualified user_id as per spec --- synapse/rest/client/v1/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 67323a16bb..3a0707c2ee 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -91,7 +91,7 @@ class LoginRestServlet(ClientV1RestServlet): password=login_submission["password"]) result = { - "user_id": login_submission["user"], # may have changed + "user_id": user_id, # may have changed "access_token": access_token, "refresh_token": refresh_token, "home_server": self.hs.hostname, -- cgit 1.5.1 From 78323ccdb359404109bfcdd8b5bf6f641ba3ff9b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 24 Aug 2015 16:17:38 +0100 Subject: Remove syutil dependency in favour of smaller single-purpose libraries --- synapse/config/key.py | 35 ++++++++++++++---------------- synapse/crypto/event_signing.py | 9 ++++---- synapse/crypto/keyring.py | 18 +++++++-------- synapse/http/client.py | 3 ++- synapse/http/matrixfederationclient.py | 4 ++-- synapse/http/server.py | 12 +++++----- synapse/python_dependencies.py | 9 +++----- synapse/rest/client/v1/voip.py | 2 +- synapse/rest/client/v2_alpha/keys.py | 3 ++- synapse/rest/key/v1/server_key_resource.py | 6 ++--- synapse/rest/key/v2/local_key_resource.py | 6 ++--- synapse/storage/event_federation.py | 2 +- synapse/storage/events.py | 23 ++++++++++---------- synapse/storage/keys.py | 2 +- synapse/storage/pusher.py | 2 +- synapse/storage/signatures.py | 2 +- synapse/storage/transactions.py | 2 +- 17 files changed, 69 insertions(+), 71 deletions(-) diff --git a/synapse/config/key.py b/synapse/config/key.py index 0494c0cb77..0f90bce04e 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -13,14 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from ._base import Config, ConfigError -import syutil.crypto.signing_key -from syutil.crypto.signing_key import ( - is_signing_algorithm_supported, decode_verify_key_bytes -) -from syutil.base64util import decode_base64 + from synapse.util.stringutils import random_string +from signedjson.key import ( + generate_signing_key, is_signing_algorithm_supported, + decode_signing_key_base64, decode_verify_key_bytes, + read_signing_keys, write_signing_keys, NACL_ED25519 +) +from unpadded_base64 import decode_base64 + +import os class KeyConfig(Config): @@ -83,9 +86,7 @@ class KeyConfig(Config): def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") try: - return syutil.crypto.signing_key.read_signing_keys( - signing_keys.splitlines(True) - ) + return read_signing_keys(signing_keys.splitlines(True)) except Exception: raise ConfigError( "Error reading signing_key." @@ -112,22 +113,18 @@ class KeyConfig(Config): if not os.path.exists(signing_key_path): with open(signing_key_path, "w") as signing_key_file: key_id = "a_" + random_string(4) - syutil.crypto.signing_key.write_signing_keys( - signing_key_file, - (syutil.crypto.signing_key.generate_signing_key(key_id),), + write_signing_keys( + signing_key_file, (generate_signing_key(key_id),), ) else: signing_keys = self.read_file(signing_key_path, "signing_key") if len(signing_keys.split("\n")[0].split()) == 1: # handle keys in the old format. key_id = "a_" + random_string(4) - key = syutil.crypto.signing_key.decode_signing_key_base64( - syutil.crypto.signing_key.NACL_ED25519, - key_id, - signing_keys.split("\n")[0] + key = decode_signing_key_base64( + NACL_ED25519, key_id, signing_keys.split("\n")[0] ) with open(signing_key_path, "w") as signing_key_file: - syutil.crypto.signing_key.write_signing_keys( - signing_key_file, - (key,), + write_signing_keys( + signing_key_file, (key,), ) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 6633b19565..64e40864af 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -15,11 +15,12 @@ # limitations under the License. -from synapse.events.utils import prune_event -from syutil.jsonutil import encode_canonical_json -from syutil.base64util import encode_base64, decode_base64 -from syutil.crypto.jsonsign import sign_json from synapse.api.errors import SynapseError, Codes +from synapse.events.utils import prune_event + +from canonicaljson import encode_canonical_json +from unpaddedbase64 import encode_base64, decode_base64 +from signedjson.sign import sign_json import hashlib import logging diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index aa74d4d0cb..a692cdbe55 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -14,20 +14,20 @@ # limitations under the License. from synapse.crypto.keyclient import fetch_server_key +from synapse.api.errors import SynapseError, Codes +from synapse.util.retryutils import get_retry_limiter +from synapse.util import unwrapFirstError +from synapse.util.async import ObservableDeferred + from twisted.internet import defer -from syutil.crypto.jsonsign import ( + +from signedjson.sign import ( verify_signed_json, signature_ids, sign_json, encode_canonical_json ) -from syutil.crypto.signing_key import ( +from signedjson.key import ( is_signing_algorithm_supported, decode_verify_key_bytes ) -from syutil.base64util import decode_base64, encode_base64 -from synapse.api.errors import SynapseError, Codes - -from synapse.util.retryutils import get_retry_limiter -from synapse.util import unwrapFirstError - -from synapse.util.async import ObservableDeferred +from unpaddedbase64 import decode_base64, encode_base64 from OpenSSL import crypto diff --git a/synapse/http/client.py b/synapse/http/client.py index 49737d55da..4b8fd3d3a3 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -15,9 +15,10 @@ from synapse.api.errors import CodeMessageException from synapse.util.logcontext import preserve_context_over_fn -from syutil.jsonutil import encode_canonical_json import synapse.metrics +from canonicaljson import encode_canonical_json + from twisted.internet import defer, reactor from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError, diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 854e17a473..1c9e552788 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -25,13 +25,13 @@ from synapse.util.async import sleep from synapse.util.logcontext import preserve_context_over_fn import synapse.metrics -from syutil.jsonutil import encode_canonical_json +from canonicaljson import encode_canonical_json from synapse.api.errors import ( SynapseError, Codes, HttpResponseException, ) -from syutil.crypto.jsonsign import sign_json +from signedjson.sign import sign_json import simplejson as json import logging diff --git a/synapse/http/server.py b/synapse/http/server.py index b60e905a62..50feea6f1c 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -21,8 +21,8 @@ from synapse.util.logcontext import LoggingContext, PreserveLoggingContext import synapse.metrics import synapse.events -from syutil.jsonutil import ( - encode_canonical_json, encode_pretty_printed_json, encode_json +from canonicaljson import ( + encode_canonical_json, encode_pretty_printed_json ) from twisted.internet import defer @@ -33,6 +33,7 @@ from twisted.web.util import redirectTo import collections import logging import urllib +import ujson logger = logging.getLogger(__name__) @@ -270,12 +271,11 @@ def respond_with_json(request, code, json_object, send_cors=False, if pretty_print: json_bytes = encode_pretty_printed_json(json_object) + "\n" else: - if canonical_json: + if canonical_json or synapse.events.USE_FROZEN_DICTS: json_bytes = encode_canonical_json(json_object) else: - json_bytes = encode_json( - json_object, using_frozen_dicts=synapse.events.USE_FROZEN_DICTS - ) + # ujson doesn't like frozen_dicts. + json_bytes = ujson.dumps(json_object, ensure_ascii=False) return respond_with_json_bytes( request, code, json_bytes, diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a87fdeb2a0..ced7ff0d7e 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -18,7 +18,9 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil>=0.0.7": ["syutil>=0.0.7"], + "unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"], + "canonicaljson>=1.0.0": ["canconicaljson>=1.0.0"], + "signedjson>=1.0.0": ["signedjson>=1.0.0"], "Twisted>=15.1.0": ["twisted>=15.1.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -54,11 +56,6 @@ def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) DEPENDENCY_LINKS = [ - github_link( - project="matrix-org/syutil", - version="v0.0.7", - egg="syutil-0.0.7", - ), github_link( project="matrix-org/matrix-angular-sdk", version="v0.6.6", diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 11d08fbced..45ed5cceac 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -40,7 +40,7 @@ class VoipRestServlet(ClientV1RestServlet): username = "%d:%s" % (expiry, auth_user.to_string()) mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1) - # We need to use standard base64 encoding here, *not* syutil's + # We need to use standard padded base64 encoding here # encode_base64 because we need to add the standard padding to get the # same result as the TURN server. password = base64.b64encode(mac.digest()) diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py index 718928eedd..21654fa2da 100644 --- a/synapse/rest/client/v2_alpha/keys.py +++ b/synapse/rest/client/v2_alpha/keys.py @@ -18,7 +18,8 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.http.servlet import RestServlet from synapse.types import UserID -from syutil.jsonutil import encode_canonical_json + +from canonicaljson import encode_canonical_json from ._base import client_v2_pattern diff --git a/synapse/rest/key/v1/server_key_resource.py b/synapse/rest/key/v1/server_key_resource.py index 71e9a51f5c..6df46969c4 100644 --- a/synapse/rest/key/v1/server_key_resource.py +++ b/synapse/rest/key/v1/server_key_resource.py @@ -16,9 +16,9 @@ from twisted.web.resource import Resource from synapse.http.server import respond_with_json_bytes -from syutil.crypto.jsonsign import sign_json -from syutil.base64util import encode_base64 -from syutil.jsonutil import encode_canonical_json +from signedjson.sign import sign_json +from unpaddedbase64 import encode_base64 +from canonicaljson import encode_canonical_json from OpenSSL import crypto import logging diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index 33cbd7cf8e..ef7699d590 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -16,9 +16,9 @@ from twisted.web.resource import Resource from synapse.http.server import respond_with_json_bytes -from syutil.crypto.jsonsign import sign_json -from syutil.base64util import encode_base64 -from syutil.jsonutil import encode_canonical_json +from signedjson.sign import sign_json +from unpaddedbase64 import encode_base64 +from canonicaljson import encode_canonical_json from hashlib import sha256 from OpenSSL import crypto import logging diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 25cc84eb95..bc90e17c63 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached -from syutil.base64util import encode_base64 +from unpaddedbase64 import encode_base64 import logging from Queue import PriorityQueue, Empty diff --git a/synapse/storage/events.py b/synapse/storage/events.py index e3eabab13d..e7439321b8 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -24,7 +24,7 @@ from synapse.util.logcontext import preserve_context_over_deferred from synapse.util.logutils import log_function from synapse.api.constants import EventTypes -from syutil.jsonutil import encode_json +from canonicaljson import encode_canonical_json from contextlib import contextmanager import logging @@ -33,6 +33,13 @@ import ujson as json logger = logging.getLogger(__name__) +def encode_json(json_object): + if USE_FROZEN_DICTS: + # ujson doesn't like frozen_dicts + return encode_canonical_json(json_object) + else: + return json.dumps(json_object, ensure_ascii=False) + # These values are used in the `enqueus_event` and `_do_fetch` methods to # control how we batch/bulk fetch events from the database. # The values are plucked out of thing air to make initial sync run faster @@ -253,8 +260,7 @@ class EventsStore(SQLBaseStore): ) metadata_json = encode_json( - event.internal_metadata.get_dict(), - using_frozen_dicts=USE_FROZEN_DICTS + event.internal_metadata.get_dict() ).decode("UTF-8") sql = ( @@ -329,12 +335,9 @@ class EventsStore(SQLBaseStore): "event_id": event.event_id, "room_id": event.room_id, "internal_metadata": encode_json( - event.internal_metadata.get_dict(), - using_frozen_dicts=USE_FROZEN_DICTS - ).decode("UTF-8"), - "json": encode_json( - event_dict(event), using_frozen_dicts=USE_FROZEN_DICTS + event.internal_metadata.get_dict() ).decode("UTF-8"), + "json": encode_json(event_dict(event)).decode("UTF-8"), } for event, _ in events_and_contexts ], @@ -353,9 +356,7 @@ class EventsStore(SQLBaseStore): "type": event.type, "processed": True, "outlier": event.internal_metadata.is_outlier(), - "content": encode_json( - event.content, using_frozen_dicts=USE_FROZEN_DICTS - ).decode("UTF-8"), + "content": encode_json(event.content).decode("UTF-8"), } for event, _ in events_and_contexts ], diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index ffd6daa880..344cacdc75 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -19,7 +19,7 @@ from synapse.util.caches.descriptors import cachedInlineCallbacks from twisted.internet import defer import OpenSSL -from syutil.crypto.signing_key import decode_verify_key_bytes +from signedjson.key import decode_verify_key_bytes import hashlib diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 08ea62681b..00b748f131 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError -from syutil.jsonutil import encode_canonical_json +from canonicaljson import encode_canonical_json import logging import simplejson as json diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 4f15e534b4..ab57b92174 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -17,7 +17,7 @@ from twisted.internet import defer from _base import SQLBaseStore -from syutil.base64util import encode_base64 +from unpaddedbase64 import encode_base64 from synapse.crypto.event_signing import compute_event_reference_hash diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index c8c7e6591a..15695e9831 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -18,7 +18,7 @@ from synapse.util.caches.descriptors import cached from collections import namedtuple -from syutil.jsonutil import encode_canonical_json +from canonicaljson import encode_canonical_json import logging logger = logging.getLogger(__name__) -- cgit 1.5.1 From f093873d697f99a59cade944bb68850b017afd09 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 24 Aug 2015 16:30:35 +0100 Subject: Replace syutil references in scripts --- scripts-dev/check_event_hash.py | 2 +- scripts-dev/check_signature.py | 8 +++----- scripts-dev/convert_server_keys.py | 8 ++++---- scripts-dev/hash_history.py | 4 ++-- scripts/upgrade_db_to_v0.6.0.py | 15 +++++---------- 5 files changed, 15 insertions(+), 22 deletions(-) diff --git a/scripts-dev/check_event_hash.py b/scripts-dev/check_event_hash.py index 679afbd268..7ccae34d48 100644 --- a/scripts-dev/check_event_hash.py +++ b/scripts-dev/check_event_hash.py @@ -1,5 +1,5 @@ from synapse.crypto.event_signing import * -from syutil.base64util import encode_base64 +from unpaddedbase64 import encode_base64 import argparse import hashlib diff --git a/scripts-dev/check_signature.py b/scripts-dev/check_signature.py index 59e3d603ac..079577908a 100644 --- a/scripts-dev/check_signature.py +++ b/scripts-dev/check_signature.py @@ -1,9 +1,7 @@ -from syutil.crypto.jsonsign import verify_signed_json -from syutil.crypto.signing_key import ( - decode_verify_key_bytes, write_signing_keys -) -from syutil.base64util import decode_base64 +from signedjson.sign import verify_signed_json +from signedjson.key import decode_verify_key_bytes, write_signing_keys +from unpaddedbase64 import decode_base64 import urllib2 import json diff --git a/scripts-dev/convert_server_keys.py b/scripts-dev/convert_server_keys.py index a1ee39059c..151551f22c 100644 --- a/scripts-dev/convert_server_keys.py +++ b/scripts-dev/convert_server_keys.py @@ -4,10 +4,10 @@ import sys import json import time import hashlib -from syutil.base64util import encode_base64 -from syutil.crypto.signing_key import read_signing_keys -from syutil.crypto.jsonsign import sign_json -from syutil.jsonutil import encode_canonical_json +from unpaddedbase64 import encode_base64 +from signedjson.key import read_signing_keys +from signedjson.sign import sign_json +from canonicaljson import encode_canonical_json def select_v1_keys(connection): diff --git a/scripts-dev/hash_history.py b/scripts-dev/hash_history.py index bdad530af8..616d6a10e7 100644 --- a/scripts-dev/hash_history.py +++ b/scripts-dev/hash_history.py @@ -6,8 +6,8 @@ from synapse.crypto.event_signing import ( add_event_pdu_content_hash, compute_pdu_event_reference_hash ) from synapse.api.events.utils import prune_pdu -from syutil.base64util import encode_base64, decode_base64 -from syutil.jsonutil import encode_canonical_json +from unpaddedbase64 import encode_base64, decode_base64 +from canonicaljson import encode_canonical_json import sqlite3 import sys diff --git a/scripts/upgrade_db_to_v0.6.0.py b/scripts/upgrade_db_to_v0.6.0.py index f466ca510e..cd4be28b86 100755 --- a/scripts/upgrade_db_to_v0.6.0.py +++ b/scripts/upgrade_db_to_v0.6.0.py @@ -4,7 +4,7 @@ from synapse.storage._base import SQLBaseStore from synapse.storage.signatures import SignatureStore from synapse.storage.event_federation import EventFederationStore -from syutil.base64util import encode_base64, decode_base64 +from unpaddedbase64 import encode_base64, decode_base64 from synapse.crypto.event_signing import compute_event_signature @@ -13,12 +13,10 @@ from synapse.events.utils import prune_event from synapse.crypto.event_signing import check_event_content_hash -from syutil.crypto.jsonsign import ( - verify_signed_json, SignatureVerifyException, -) -from syutil.crypto.signing_key import decode_verify_key_bytes +from signedjson.sign import verify_signed_json, SignatureVerifyException +from signedjson.key import decode_verify_key_bytes -from syutil.jsonutil import encode_canonical_json +from canonicaljson import encode_canonical_json import argparse # import dns.resolver @@ -26,7 +24,6 @@ import hashlib import httplib import json import sqlite3 -import syutil import urllib2 @@ -324,8 +321,6 @@ if __name__ == "__main__": ) args = parser.parse_args() - signing_key = syutil.crypto.signing_key.read_signing_keys( - args.signing_key - ) + signing_key = signedjson.key.read_signing_keys(args.signing_key) main(args.database, args.server_name, signing_key[0]) -- cgit 1.5.1 From 01fc3943f1aa409766f8fc54037af9102c55c658 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 24 Aug 2015 17:18:58 +0100 Subject: Fix indent --- synapse/storage/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index e7439321b8..fba837f461 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -35,7 +35,7 @@ logger = logging.getLogger(__name__) def encode_json(json_object): if USE_FROZEN_DICTS: - # ujson doesn't like frozen_dicts + # ujson doesn't like frozen_dicts return encode_canonical_json(json_object) else: return json.dumps(json_object, ensure_ascii=False) -- cgit 1.5.1 From 037481a033aeb80b86a2e7e074e29cfaf8c23ea8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 24 Aug 2015 17:48:57 +0100 Subject: Remove autoincrement since we incrementing the ID in the storage layer --- synapse/storage/schema/delta/23/refresh_tokens.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/23/refresh_tokens.sql b/synapse/storage/schema/delta/23/refresh_tokens.sql index 46839e016c..437b1ac1be 100644 --- a/synapse/storage/schema/delta/23/refresh_tokens.sql +++ b/synapse/storage/schema/delta/23/refresh_tokens.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS refresh_tokens( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, token TEXT NOT NULL, user_id TEXT NOT NULL, UNIQUE (token) -- cgit 1.5.1 From cf8c04948f9a8e8f50349918622164abd7590a13 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 25 Aug 2015 10:42:59 +0100 Subject: Fix typo in module imports and package dependencies --- synapse/config/key.py | 2 +- synapse/python_dependencies.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/key.py b/synapse/config/key.py index 0f90bce04e..23ac8a3fca 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -21,7 +21,7 @@ from signedjson.key import ( decode_signing_key_base64, decode_verify_key_bytes, read_signing_keys, write_signing_keys, NACL_ED25519 ) -from unpadded_base64 import decode_base64 +from unpaddedbase64 import decode_base64 import os diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ced7ff0d7e..4c07aa07aa 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"], - "canonicaljson>=1.0.0": ["canconicaljson>=1.0.0"], + "canonicaljson>=1.0.0": ["canonicaljson>=1.0.0"], "signedjson>=1.0.0": ["signedjson>=1.0.0"], "Twisted>=15.1.0": ["twisted>=15.1.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], -- cgit 1.5.1 From 2e31dd2ad3bdb67a2b66ad1227110c1c29a1add6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 24 Aug 2015 17:17:31 +0100 Subject: Add tox.ini file for synapse --- MANIFEST.in | 5 +++++ setup.py | 5 ----- tox.ini | 25 +++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 tox.ini diff --git a/MANIFEST.in b/MANIFEST.in index a9b543af82..d1b6b69633 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,4 +12,9 @@ recursive-include demo *.py recursive-include demo *.sh recursive-include docs * recursive-include scripts * +recursive-include scripts-dev * recursive-include tests *.py + +recursive-include static *.css +recursive-include static *.html +recursive-include static *.js diff --git a/setup.py b/setup.py index 16ccc0f1b8..8ad20df7cb 100755 --- a/setup.py +++ b/setup.py @@ -47,11 +47,6 @@ setup( packages=find_packages(exclude=["tests", "tests.*"]), description="Reference Synapse Home Server", install_requires=dependencies['requirements'](include_conditional=True).keys(), - setup_requires=[ - "Twisted>=15.1.0", # Here to override setuptools_trial's dependency on Twisted>=2.4.0 - "setuptools_trial", - "mock" - ], dependency_links=dependencies["DEPENDENCY_LINKS"], include_package_data=True, zip_safe=False, diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000..a1e0ac70d3 --- /dev/null +++ b/tox.ini @@ -0,0 +1,25 @@ +[tox] +envlist = packaging, py27, pep8 + +[testenv] +deps = + coverage + Twisted>=15.1 + mock +commands = + coverage run --source=./synapse {envbindir}/trial tests + coverage report -m +install_command = + pip install --process-dependency-links --pre {opts} {packages} + +[testenv:packaging] +deps = + check-manifest +commands = + check-manifest + +[testenv:pep8] +basepython = python2.7 +deps = + flake8 +commands = flake8 synapse -- cgit 1.5.1 From 12b1a47ba4a4f7eb8f2745a3ef3c5c90e9197b0e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 25 Aug 2015 14:22:33 +0100 Subject: Only include demo/demo.tls.dh. Don't include any other dh file --- MANIFEST.in | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index d1b6b69633..621e34cb76 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,13 +3,13 @@ include LICENSE include VERSION include *.rst include demo/README +include demo/demo.tls.dh +include demo/*.py +include demo/*.sh recursive-include synapse/storage/schema *.sql recursive-include synapse/storage/schema *.py -recursive-include demo *.dh -recursive-include demo *.py -recursive-include demo *.sh recursive-include docs * recursive-include scripts * recursive-include scripts-dev * @@ -18,3 +18,5 @@ recursive-include tests *.py recursive-include static *.css recursive-include static *.html recursive-include static *.js + +prune demo/etc -- cgit 1.5.1 -- cgit 1.5.1 -- cgit 1.5.1 -- cgit 1.5.1 From d046adf4ecf0737dc111a93d1f1ae7f41bc1bccb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 25 Aug 2015 15:44:05 +0100 Subject: Set PYTHONDONTWRITEBYTECODE in the tox environment so that we don't spew .pyc files everywhere --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index a1e0ac70d3..58cf9613cb 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,8 @@ deps = coverage Twisted>=15.1 mock +setenv = + PYTHONDONTWRITEBYTECODE = no_byte_code commands = coverage run --source=./synapse {envbindir}/trial tests coverage report -m -- cgit 1.5.1 From 3a20cdcd27fa0006ff15156ef07602c816d86643 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 25 Aug 2015 15:45:03 +0100 Subject: Add .tox to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 960183a794..4c336b710d 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ build/ localhost-800*/ static/client/register/register_config.js +.tox -- cgit 1.5.1 From a0b181bd17cb7ec2a43ed2dbdeb1bb40f3f4373c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 25 Aug 2015 16:23:06 +0100 Subject: Remove completely unused concepts from codebase Removes device_id and ClientInfo device_id is never actually written, and the matrix.org DB has no non-null entries for it. Right now, it's just cluttering up code. This doesn't remove the columns from the database, because that's fiddly. --- synapse/api/auth.py | 17 ++++++--------- synapse/handlers/admin.py | 1 + synapse/handlers/message.py | 9 +++----- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/directory.py | 4 ++-- synapse/rest/client/v1/events.py | 4 ++-- synapse/rest/client/v1/initial_sync.py | 2 +- synapse/rest/client/v1/presence.py | 8 +++---- synapse/rest/client/v1/profile.py | 4 ++-- synapse/rest/client/v1/pusher.py | 4 ++-- synapse/rest/client/v1/room.py | 34 ++++++++++++++--------------- synapse/rest/client/v1/voip.py | 2 +- synapse/rest/client/v2_alpha/account.py | 4 ++-- synapse/rest/client/v2_alpha/filter.py | 4 ++-- synapse/rest/client/v2_alpha/keys.py | 6 ++--- synapse/rest/client/v2_alpha/receipts.py | 2 +- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/rest/media/v1/upload_resource.py | 2 +- synapse/storage/__init__.py | 7 +++--- synapse/storage/registration.py | 5 ++--- synapse/types.py | 4 ---- tests/api/test_auth.py | 8 +++---- tests/rest/client/v1/test_presence.py | 2 -- tests/rest/client/v1/test_rooms.py | 7 ------ tests/rest/client/v1/test_typing.py | 1 - tests/rest/client/v2_alpha/__init__.py | 1 - tests/storage/test_registration.py | 2 -- tests/utils.py | 3 +-- 29 files changed, 63 insertions(+), 90 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3d9237ccc3..1496db7dff 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -20,7 +20,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError from synapse.util.logutils import log_function -from synapse.types import UserID, ClientInfo +from synapse.types import UserID import logging @@ -322,9 +322,9 @@ class Auth(object): Args: request - An HTTP request with an access_token query parameter. Returns: - tuple : of UserID and device string: - User ID object of the user making the request - ClientInfo object of the client instance the user is using + tuple of: + UserID (str) + Access token ID (str) Raises: AuthError if no user by that token exists or the token is invalid. """ @@ -355,7 +355,7 @@ class Auth(object): request.authenticated_entity = user_id defer.returnValue( - (UserID.from_string(user_id), ClientInfo("", "")) + (UserID.from_string(user_id), "") ) return except KeyError: @@ -363,7 +363,6 @@ class Auth(object): user_info = yield self.get_user_by_access_token(access_token) user = user_info["user"] - device_id = user_info["device_id"] token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) @@ -375,14 +374,13 @@ class Auth(object): self.store.insert_client_ip( user=user, access_token=access_token, - device_id=user_info["device_id"], ip=ip_addr, user_agent=user_agent ) request.authenticated_entity = user.to_string() - defer.returnValue((user, ClientInfo(device_id, token_id))) + defer.returnValue((user, token_id,)) except KeyError: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", @@ -396,7 +394,7 @@ class Auth(object): Args: token (str): The access token to get the user by. Returns: - dict : dict that includes the user, device_id, and whether the + dict : dict that includes the user and whether the user is a server admin. Raises: AuthError if no user by that token exists or the token is invalid. @@ -409,7 +407,6 @@ class Auth(object): ) user_info = { "admin": bool(ret.get("admin", False)), - "device_id": ret.get("device_id"), "user": UserID.from_string(ret.get("name")), "token_id": ret.get("token_id", None), } diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 1c9e7152c7..d852a18555 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -34,6 +34,7 @@ class AdminHandler(BaseHandler): d = {} for r in res: + # Note that device_id is always None device = d.setdefault(r["device_id"], {}) session = device.setdefault(r["access_token"], []) session.append({ diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index f12465fa2c..23b779ad7c 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -183,7 +183,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def create_and_send_event(self, event_dict, ratelimit=True, - client=None, txn_id=None): + token_id=None, txn_id=None): """ Given a dict from a client, create and handle a new event. Creates an FrozenEvent object, filling out auth_events, prev_events, @@ -217,11 +217,8 @@ class MessageHandler(BaseHandler): builder.content ) - if client is not None: - if client.token_id is not None: - builder.internal_metadata.token_id = client.token_id - if client.device_id is not None: - builder.internal_metadata.device_id = client.device_id + if token_id is not None: + builder.internal_metadata.token_id = token_id if txn_id is not None: builder.internal_metadata.txn_id = txn_id diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 2ce754b028..504b63eab4 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -31,7 +31,7 @@ class WhoisRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) if not is_admin and target_user != auth_user: diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 6758a888b3..4dcda57c1b 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -69,7 +69,7 @@ class ClientDirectoryServer(ClientV1RestServlet): try: # try to auth as a user - user, client = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) try: user_id = user.to_string() yield dir_handler.create_association( @@ -116,7 +116,7 @@ class ClientDirectoryServer(ClientV1RestServlet): # fallback to default user behaviour if they aren't an AS pass - user, client = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index 77b7c25a03..582148b659 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -34,7 +34,7 @@ class EventStreamRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) try: handler = self.handlers.event_stream_handler pagin_config = PaginationConfig.from_request(request) @@ -71,7 +71,7 @@ class EventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, event_id): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 4a259bba64..4ea4da653c 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -25,7 +25,7 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user, client = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 78d4f2b128..a770efd841 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -32,7 +32,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( @@ -42,7 +42,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = {} @@ -77,7 +77,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): @@ -97,7 +97,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 1e77eb49cf..fdde88a60d 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -37,7 +37,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: @@ -70,7 +70,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index c83287c028..3aabc93b8b 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -27,7 +27,7 @@ class PusherRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - user, client = yield self.auth.get_user_by_req(request) + user, token_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -65,7 +65,7 @@ class PusherRestServlet(ClientV1RestServlet): try: yield pusher_pool.add_pusher( user_name=user.to_string(), - access_token=client.token_id, + access_token=token_id, profile_tag=content['profile_tag'], kind=content['kind'], app_id=content['app_id'], diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index b4a70cba99..c9c27dd5a0 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -62,7 +62,7 @@ class RoomCreateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) room_config = self.get_room_config(request) info = yield self.make_room(room_config, auth_user, None) @@ -125,7 +125,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): - user, client = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) msg_handler = self.handlers.message_handler data = yield msg_handler.get_room_data( @@ -143,7 +143,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, event_type, state_key, txn_id=None): - user, client = yield self.auth.get_user_by_req(request) + user, token_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -159,7 +159,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): msg_handler = self.handlers.message_handler yield msg_handler.create_and_send_event( - event_dict, client=client, txn_id=txn_id, + event_dict, token_id=token_id, txn_id=txn_id, ) defer.returnValue((200, {})) @@ -175,7 +175,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, event_type, txn_id=None): - user, client = yield self.auth.get_user_by_req(request) + user, token_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -186,7 +186,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): "room_id": room_id, "sender": user.to_string(), }, - client=client, + token_id=token_id, txn_id=txn_id, ) @@ -220,7 +220,7 @@ class JoinRoomAliasServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_identifier, txn_id=None): - user, client = yield self.auth.get_user_by_req(request) + user, token_id = yield self.auth.get_user_by_req(request) # the identifier could be a room alias or a room id. Try one then the # other if it fails to parse, without swallowing other valid @@ -250,7 +250,7 @@ class JoinRoomAliasServlet(ClientV1RestServlet): "sender": user.to_string(), "state_key": user.to_string(), }, - client=client, + token_id=token_id, txn_id=txn_id, ) @@ -289,7 +289,7 @@ class RoomMemberListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) - user, client = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) handler = self.handlers.room_member_handler members = yield handler.get_room_members_as_pagination_chunk( room_id=room_id, @@ -317,7 +317,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, client = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request( request, default_limit=10, ) @@ -341,7 +341,7 @@ class RoomStateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, client = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) handler = self.handlers.message_handler # Get all the current state for this room events = yield handler.get_state_events( @@ -357,7 +357,7 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, client = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, @@ -402,7 +402,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, membership_action, txn_id=None): - user, client = yield self.auth.get_user_by_req(request) + user, token_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -427,7 +427,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): "sender": user.to_string(), "state_key": state_key, }, - client=client, + token_id=token_id, txn_id=txn_id, ) @@ -457,7 +457,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, event_id, txn_id=None): - user, client = yield self.auth.get_user_by_req(request) + user, token_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -469,7 +469,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): "sender": user.to_string(), "redacts": event_id, }, - client=client, + token_id=token_id, txn_id=txn_id, ) @@ -497,7 +497,7 @@ class RoomTypingRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, user_id): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) target_user = UserID.from_string(urllib.unquote(user_id)) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 11d08fbced..4ae2d81b70 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -28,7 +28,7 @@ class VoipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) turnUris = self.hs.config.turn_uris turnSecret = self.hs.config.turn_shared_secret diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 522a312c9e..b5edffdb60 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -55,7 +55,7 @@ class PasswordRestServlet(RestServlet): if LoginType.PASSWORD in result: # if using password, they should also be logged in - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) if auth_user.to_string() != result[LoginType.PASSWORD]: raise LoginError(400, "", Codes.UNKNOWN) user_id = auth_user.to_string() @@ -119,7 +119,7 @@ class ThreepidRestServlet(RestServlet): raise SynapseError(400, "Missing param", Codes.MISSING_PARAM) threePidCreds = body['threePidCreds'] - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) threepid = yield self.identity_handler.threepid_from_creds(threePidCreds) diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 703250cea8..f8f91b63f5 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -40,7 +40,7 @@ class GetFilterRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id, filter_id): target_user = UserID.from_string(user_id) - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) if target_user != auth_user: raise AuthError(403, "Cannot get filters for other users") @@ -76,7 +76,7 @@ class CreateFilterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) if target_user != auth_user: raise AuthError(403, "Cannot create filters for other users") diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py index 718928eedd..ec1145454f 100644 --- a/synapse/rest/client/v2_alpha/keys.py +++ b/synapse/rest/client/v2_alpha/keys.py @@ -63,7 +63,7 @@ class KeyUploadServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, device_id): - auth_user, client_info = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) user_id = auth_user.to_string() # TODO: Check that the device_id matches that in the authentication # or derive the device_id from the authentication instead. @@ -108,7 +108,7 @@ class KeyUploadServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, device_id): - auth_user, client_info = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) user_id = auth_user.to_string() result = yield self.store.count_e2e_one_time_keys(user_id, device_id) @@ -180,7 +180,7 @@ class KeyQueryServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id, device_id): - auth_user, client_info = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) auth_user_id = auth_user.to_string() user_id = user_id if user_id else auth_user_id device_ids = [device_id] if device_id else [] diff --git a/synapse/rest/client/v2_alpha/receipts.py b/synapse/rest/client/v2_alpha/receipts.py index 40406e2ede..52e99f54d5 100644 --- a/synapse/rest/client/v2_alpha/receipts.py +++ b/synapse/rest/client/v2_alpha/receipts.py @@ -39,7 +39,7 @@ class ReceiptRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, receipt_type, event_id): - user, client = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) yield self.receipts_handler.received_client_receipt( room_id, diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index f2fd0b9f32..83a257b969 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -87,7 +87,7 @@ class SyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user, client = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) timeout = parse_integer(request, "timeout", default=0) limit = parse_integer(request, "limit", required=True) diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index e77a20fb2e..c28dc86cd7 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -66,7 +66,7 @@ class ContentRepoResource(resource.Resource): @defer.inlineCallbacks def map_request_to_name(self, request): # auth the user - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) # namespace all file uploads on the user prefix = base64.urlsafe_b64encode( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index cdd1d44e07..439d5a30a8 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -70,7 +70,7 @@ class UploadResource(BaseMediaResource): @request_handler @defer.inlineCallbacks def _async_render_POST(self, request): - auth_user, client = yield self.auth.get_user_by_req(request) + auth_user, _ = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 53673b3bf5..77cb1dbd81 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -94,9 +94,9 @@ class DataStore(RoomMemberStore, RoomStore, ) @defer.inlineCallbacks - def insert_client_ip(self, user, access_token, device_id, ip, user_agent): + def insert_client_ip(self, user, access_token, ip, user_agent): now = int(self._clock.time_msec()) - key = (user.to_string(), access_token, device_id, ip) + key = (user.to_string(), access_token, ip) try: last_seen = self.client_ip_last_seen.get(key) @@ -120,7 +120,6 @@ class DataStore(RoomMemberStore, RoomStore, "user_agent": user_agent, }, values={ - "device_id": device_id, "last_seen": now, }, desc="insert_client_ip", @@ -132,7 +131,7 @@ class DataStore(RoomMemberStore, RoomStore, table="user_ips", keyvalues={"user_id": user.to_string()}, retcols=[ - "device_id", "access_token", "ip", "user_agent", "last_seen" + "access_token", "ip", "user_agent", "last_seen" ], desc="get_user_ip_and_agents", ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f632306688..240d14c4d0 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -163,7 +163,7 @@ class RegistrationStore(SQLBaseStore): Args: token (str): The access token of a user. Returns: - dict: Including the name (user_id), device_id and whether they are + dict: Including the name (user_id) and whether they are an admin. Raises: StoreError if no user was found. @@ -228,8 +228,7 @@ class RegistrationStore(SQLBaseStore): def _query_for_auth(self, txn, token): sql = ( - "SELECT users.name, users.admin," - " access_tokens.device_id, access_tokens.id as token_id" + "SELECT users.name, users.admin, access_tokens.id as token_id" " FROM users" " INNER JOIN access_tokens on users.name = access_tokens.user_id" " WHERE token = ?" diff --git a/synapse/types.py b/synapse/types.py index e190374cbd..9cffc33d27 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -209,7 +209,3 @@ class RoomStreamToken(namedtuple("_StreamToken", "topological stream")): return "t%d-%d" % (self.topological, self.stream) else: return "s%d" % (self.stream,) - - -# token_id is the primary key ID of the access token, not the access token itself. -ClientInfo = namedtuple("ClientInfo", ("device_id", "token_id")) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 3343c635cc..777eb0395e 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -40,7 +40,6 @@ class AuthTestCase(unittest.TestCase): self.store.get_app_service_by_token = Mock(return_value=None) user_info = { "name": self.test_user, - "device_id": "nothing", "token_id": "ditto", "admin": False } @@ -49,7 +48,7 @@ class AuthTestCase(unittest.TestCase): request = Mock(args={}) request.args["access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = Mock(return_value=[""]) - (user, info) = yield self.auth.get_user_by_req(request) + (user, _) = yield self.auth.get_user_by_req(request) self.assertEquals(user.to_string(), self.test_user) def test_get_user_by_req_user_bad_token(self): @@ -66,7 +65,6 @@ class AuthTestCase(unittest.TestCase): self.store.get_app_service_by_token = Mock(return_value=None) user_info = { "name": self.test_user, - "device_id": "nothing", "token_id": "ditto", "admin": False } @@ -86,7 +84,7 @@ class AuthTestCase(unittest.TestCase): request = Mock(args={}) request.args["access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = Mock(return_value=[""]) - (user, info) = yield self.auth.get_user_by_req(request) + (user, _) = yield self.auth.get_user_by_req(request) self.assertEquals(user.to_string(), self.test_user) def test_get_user_by_req_appservice_bad_token(self): @@ -121,7 +119,7 @@ class AuthTestCase(unittest.TestCase): request.args["access_token"] = [self.test_token] request.args["user_id"] = [masquerading_user_id] request.requestHeaders.getRawHeaders = Mock(return_value=[""]) - (user, info) = yield self.auth.get_user_by_req(request) + (user, _) = yield self.auth.get_user_by_req(request) self.assertEquals(user.to_string(), masquerading_user_id) def test_get_user_by_req_appservice_valid_token_bad_user_id(self): diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 0b78a82a66..4039a86d85 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -74,7 +74,6 @@ class PresenceStateTestCase(unittest.TestCase): return { "user": UserID.from_string(myid), "admin": False, - "device_id": None, "token_id": 1, } @@ -163,7 +162,6 @@ class PresenceListTestCase(unittest.TestCase): return { "user": UserID.from_string(myid), "admin": False, - "device_id": None, "token_id": 1, } diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 2e55cc08a1..dd1e67e0f9 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -58,7 +58,6 @@ class RoomPermissionsTestCase(RestTestCase): return { "user": UserID.from_string(self.auth_user_id), "admin": False, - "device_id": None, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token @@ -445,7 +444,6 @@ class RoomsMemberListTestCase(RestTestCase): return { "user": UserID.from_string(self.auth_user_id), "admin": False, - "device_id": None, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token @@ -525,7 +523,6 @@ class RoomsCreateTestCase(RestTestCase): return { "user": UserID.from_string(self.auth_user_id), "admin": False, - "device_id": None, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token @@ -618,7 +615,6 @@ class RoomTopicTestCase(RestTestCase): return { "user": UserID.from_string(self.auth_user_id), "admin": False, - "device_id": None, "token_id": 1, } @@ -725,7 +721,6 @@ class RoomMemberStateTestCase(RestTestCase): return { "user": UserID.from_string(self.auth_user_id), "admin": False, - "device_id": None, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token @@ -852,7 +847,6 @@ class RoomMessagesTestCase(RestTestCase): return { "user": UserID.from_string(self.auth_user_id), "admin": False, - "device_id": None, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token @@ -949,7 +943,6 @@ class RoomInitialSyncTestCase(RestTestCase): return { "user": UserID.from_string(self.auth_user_id), "admin": False, - "device_id": None, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index dc8bbaaf0e..0f70ce81dc 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -65,7 +65,6 @@ class RoomTypingTestCase(RestTestCase): return { "user": UserID.from_string(self.auth_user_id), "admin": False, - "device_id": None, "token_id": 1, } diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index 15568b36cd..badb59f080 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -47,7 +47,6 @@ class V2AlphaRestTestCase(unittest.TestCase): return { "user": UserID.from_string(self.USER_ID), "admin": False, - "device_id": None, "token_id": 1, } hs.get_auth().get_user_by_access_token = _get_user_by_access_token diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index a4f929796a..54fe10d58f 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -54,7 +54,6 @@ class RegistrationStoreTestCase(unittest.TestCase): self.assertDictContainsSubset( { "admin": 0, - "device_id": None, "name": self.user_id, }, result @@ -72,7 +71,6 @@ class RegistrationStoreTestCase(unittest.TestCase): self.assertDictContainsSubset( { "admin": 0, - "device_id": None, "name": self.user_id, }, result diff --git a/tests/utils.py b/tests/utils.py index d0fba2252d..ff560ef342 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -282,7 +282,6 @@ class MemoryDataStore(object): return { "name": self.tokens_to_users[token], "admin": 0, - "device_id": None, } except: raise StoreError(400, "User does not exist.") @@ -380,7 +379,7 @@ class MemoryDataStore(object): def get_ops_levels(self, room_id): return defer.succeed((5, 5, 5)) - def insert_client_ip(self, user, device_id, access_token, ip, user_agent): + def insert_client_ip(self, user, access_token, ip, user_agent): return defer.succeed(None) -- cgit 1.5.1 From 57619d6058fc5f70fc5c799d0fc7f362b0a03bcf Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 25 Aug 2015 16:25:46 +0100 Subject: Re-wrap line --- synapse/api/auth.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 1496db7dff..b41e34e658 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -354,9 +354,7 @@ class Auth(object): request.authenticated_entity = user_id - defer.returnValue( - (UserID.from_string(user_id), "") - ) + defer.returnValue((UserID.from_string(user_id), "")) return except KeyError: pass # normal users won't have the user_id query parameter set. -- cgit 1.5.1 From a9d8bd95e722e24c7ddd6b14a3714c1b2f737d4d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 25 Aug 2015 16:29:39 +0100 Subject: Stop looking up "admin", which we never read --- synapse/api/auth.py | 4 +--- synapse/storage/registration.py | 5 ++--- tests/api/test_auth.py | 2 -- tests/rest/client/v1/test_presence.py | 2 -- tests/rest/client/v1/test_rooms.py | 7 ------- tests/rest/client/v1/test_typing.py | 1 - tests/rest/client/v2_alpha/__init__.py | 1 - tests/storage/test_registration.py | 6 ++---- tests/utils.py | 1 - 9 files changed, 5 insertions(+), 24 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index b41e34e658..65ee1452ce 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -392,8 +392,7 @@ class Auth(object): Args: token (str): The access token to get the user by. Returns: - dict : dict that includes the user and whether the - user is a server admin. + dict : dict that includes the user and the ID of their access token. Raises: AuthError if no user by that token exists or the token is invalid. """ @@ -404,7 +403,6 @@ class Auth(object): errcode=Codes.UNKNOWN_TOKEN ) user_info = { - "admin": bool(ret.get("admin", False)), "user": UserID.from_string(ret.get("name")), "token_id": ret.get("token_id", None), } diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 240d14c4d0..a2d0f7c4b1 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -163,8 +163,7 @@ class RegistrationStore(SQLBaseStore): Args: token (str): The access token of a user. Returns: - dict: Including the name (user_id) and whether they are - an admin. + dict: Including the name (user_id) and the ID of their access token. Raises: StoreError if no user was found. """ @@ -228,7 +227,7 @@ class RegistrationStore(SQLBaseStore): def _query_for_auth(self, txn, token): sql = ( - "SELECT users.name, users.admin, access_tokens.id as token_id" + "SELECT users.name, access_tokens.id as token_id" " FROM users" " INNER JOIN access_tokens on users.name = access_tokens.user_id" " WHERE token = ?" diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 777eb0395e..22fc804331 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -41,7 +41,6 @@ class AuthTestCase(unittest.TestCase): user_info = { "name": self.test_user, "token_id": "ditto", - "admin": False } self.store.get_user_by_access_token = Mock(return_value=user_info) @@ -66,7 +65,6 @@ class AuthTestCase(unittest.TestCase): user_info = { "name": self.test_user, "token_id": "ditto", - "admin": False } self.store.get_user_by_access_token = Mock(return_value=user_info) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 4039a86d85..91547bdd06 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -73,7 +73,6 @@ class PresenceStateTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(myid), - "admin": False, "token_id": 1, } @@ -161,7 +160,6 @@ class PresenceListTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(myid), - "admin": False, "token_id": 1, } diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index dd1e67e0f9..34ab47d02e 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -57,7 +57,6 @@ class RoomPermissionsTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), - "admin": False, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token @@ -443,7 +442,6 @@ class RoomsMemberListTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), - "admin": False, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token @@ -522,7 +520,6 @@ class RoomsCreateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), - "admin": False, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token @@ -614,7 +611,6 @@ class RoomTopicTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), - "admin": False, "token_id": 1, } @@ -720,7 +716,6 @@ class RoomMemberStateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), - "admin": False, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token @@ -846,7 +841,6 @@ class RoomMessagesTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), - "admin": False, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token @@ -942,7 +936,6 @@ class RoomInitialSyncTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), - "admin": False, "token_id": 1, } hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 0f70ce81dc..1c4519406d 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -64,7 +64,6 @@ class RoomTypingTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.auth_user_id), - "admin": False, "token_id": 1, } diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index badb59f080..ef972a53aa 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -46,7 +46,6 @@ class V2AlphaRestTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { "user": UserID.from_string(self.USER_ID), - "admin": False, "token_id": 1, } hs.get_auth().get_user_by_access_token = _get_user_by_access_token diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 54fe10d58f..0cce6c37df 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -53,8 +53,7 @@ class RegistrationStoreTestCase(unittest.TestCase): self.assertDictContainsSubset( { - "admin": 0, - "name": self.user_id, + "name": self.user_id, }, result ) @@ -70,8 +69,7 @@ class RegistrationStoreTestCase(unittest.TestCase): self.assertDictContainsSubset( { - "admin": 0, - "name": self.user_id, + "name": self.user_id, }, result ) diff --git a/tests/utils.py b/tests/utils.py index ff560ef342..3766a994f2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -281,7 +281,6 @@ class MemoryDataStore(object): try: return { "name": self.tokens_to_users[token], - "admin": 0, } except: raise StoreError(400, "User does not exist.") -- cgit 1.5.1 From 825f0875bc266375cafd56067cf895575c0d8893 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 25 Aug 2015 16:37:37 +0100 Subject: Fix up one more reference --- synapse/handlers/sync.py | 1 - synapse/rest/client/v2_alpha/sync.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 353a416054..9914ff6f9c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -28,7 +28,6 @@ logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", - "client_info", "limit", "gap", "sort", diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 83a257b969..cac28b47b6 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -87,7 +87,7 @@ class SyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user, _ = yield self.auth.get_user_by_req(request) + user, token_id = yield self.auth.get_user_by_req(request) timeout = parse_integer(request, "timeout", default=0) limit = parse_integer(request, "limit", required=True) @@ -125,7 +125,6 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, - client_info=client, gap=gap, limit=limit, sort=sort, @@ -152,7 +151,7 @@ class SyncRestServlet(RestServlet): sync_result.private_user_data, filter, time_now ), "rooms": self.encode_rooms( - sync_result.rooms, filter, time_now, client.token_id + sync_result.rooms, filter, time_now, token_id ), "next_batch": sync_result.next_batch.to_string(), } -- cgit 1.5.1 From e768d7b3a6f925dbd7c5466fa86a4a21432d0d1d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 26 Aug 2015 11:41:42 +0100 Subject: Allow tests to be filter when using tox `tox` will run all tests `tox tests.api.test_auth.AuthTestCase` will run just the tests in AuthTestCase --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 58cf9613cb..370a63cbb8 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ deps = setenv = PYTHONDONTWRITEBYTECODE = no_byte_code commands = - coverage run --source=./synapse {envbindir}/trial tests + coverage run --source=./synapse {envbindir}/trial {posargs:tests} coverage report -m install_command = pip install --process-dependency-links --pre {opts} {packages} -- cgit 1.5.1 From 4f6fa981ec9035b2708ba0dc9db21d5dd629f389 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 26 Aug 2015 12:45:29 +0100 Subject: Make 'setup.py test' run tox --- setup.cfg | 3 --- setup.py | 27 ++++++++++++++++++++++++++- tox.ini | 2 +- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index abb649958e..ba027c7d13 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,9 +3,6 @@ source-dir = docs/sphinx build-dir = docs/build all_files = 1 -[aliases] -test = trial - [trial] test_suite = tests diff --git a/setup.py b/setup.py index 8ad20df7cb..adc3316928 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,8 @@ import glob import os -from setuptools import setup, find_packages +from setuptools import setup, find_packages, Command +import sys here = os.path.abspath(os.path.dirname(__file__)) @@ -37,6 +38,28 @@ def exec_file(path_segments): exec(code, result) return result + +class Tox(Command): + user_options = [('tox-args=', 'a', "Arguments to pass to tox")] + + def initialize_options(self): + self.tox_args = None + + def finalize_options(self): + self.test_args = [] + self.test_suite = True + + def run(self): + #import here, cause outside the eggs aren't loaded + import tox + import shlex + args = self.tox_args + if args: + args = shlex.split(self.tox_args) + errno = tox.cmdline(args=args) + sys.exit(errno) + + version = exec_file(("synapse", "__init__.py"))["__version__"] dependencies = exec_file(("synapse", "python_dependencies.py")) long_description = read_file(("README.rst",)) @@ -52,4 +75,6 @@ setup( zip_safe=False, long_description=long_description, scripts=["synctl"] + glob.glob("scripts/*"), + tests_require=['tox'], + cmdclass={'test': Tox}, ) diff --git a/tox.ini b/tox.ini index 370a63cbb8..3f0649b628 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ deps = setenv = PYTHONDONTWRITEBYTECODE = no_byte_code commands = - coverage run --source=./synapse {envbindir}/trial {posargs:tests} + coverage run --source=synapse {envbindir}/trial {posargs:tests} coverage report -m install_command = pip install --process-dependency-links --pre {opts} {packages} -- cgit 1.5.1 From fd246fde8934f1461589df20efcf53ae1bd0e90d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 26 Aug 2015 12:59:02 +0100 Subject: Install tox locally if it wasn't already installed when running setup.py test --- setup.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index adc3316928..4a7b3c99aa 100755 --- a/setup.py +++ b/setup.py @@ -51,7 +51,16 @@ class Tox(Command): def run(self): #import here, cause outside the eggs aren't loaded - import tox + try: + import tox + except ImportError: + try: + self.distribution.fetch_build_eggs("tox") + import tox + except: + raise RuntimeError( + "The tests need 'tox' to run. Please install 'tox'." + ) import shlex args = self.tox_args if args: @@ -75,6 +84,5 @@ setup( zip_safe=False, long_description=long_description, scripts=["synctl"] + glob.glob("scripts/*"), - tests_require=['tox'], cmdclass={'test': Tox}, ) -- cgit 1.5.1 From 06094591c5397af64a495dce2271da2df5b491b9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 26 Aug 2015 13:13:01 +0100 Subject: Pass an empty list of arguments to tox if no arguments are given --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 4a7b3c99aa..0e3f2f18d5 100755 --- a/setup.py +++ b/setup.py @@ -65,6 +65,8 @@ class Tox(Command): args = self.tox_args if args: args = shlex.split(self.tox_args) + else: + args = [] errno = tox.cmdline(args=args) sys.exit(errno) -- cgit 1.5.1 From 6a4b650d8ad3e6c095020cac3861e430d643d53d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 26 Aug 2015 13:22:23 +0100 Subject: Attempt to validate macaroons A couple of weird caveats: * If we can't validate your macaroon, we fall back to checking that your access token is in the DB, and ignoring the failure * Even if we can validate your macaroon, we still have to hit the DB to get the access token ID, which we pretend is a device ID all over the codebase. This mostly adds the interesting code, and points out the two pieces we need to delete (and necessary conditions) in order to fix the above caveats. --- synapse/api/auth.py | 104 +++++++++++++++++++++--- tests/api/test_auth.py | 142 ++++++++++++++++++++++++++++++++- tests/rest/client/v1/test_presence.py | 8 +- tests/rest/client/v1/test_rooms.py | 28 +++---- tests/rest/client/v1/test_typing.py | 4 +- tests/rest/client/v1/utils.py | 3 - tests/rest/client/v2_alpha/__init__.py | 4 +- 7 files changed, 257 insertions(+), 36 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 65ee1452ce..f8ea1e2c69 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -23,6 +23,7 @@ from synapse.util.logutils import log_function from synapse.types import UserID import logging +import pymacaroons logger = logging.getLogger(__name__) @@ -40,6 +41,12 @@ class Auth(object): self.store = hs.get_datastore() self.state = hs.get_state_handler() self.TOKEN_NOT_FOUND_HTTP_STATUS = 401 + self._KNOWN_CAVEAT_PREFIXES = set([ + "gen = ", + "type = ", + "time < ", + "user_id = ", + ]) def check(self, event, auth_events): """ Checks if this event is correctly authed. @@ -359,8 +366,8 @@ class Auth(object): except KeyError: pass # normal users won't have the user_id query parameter set. - user_info = yield self.get_user_by_access_token(access_token) - user = user_info["user"] + user_info = yield self._get_user_by_access_token(access_token) + user_id = user_info["user_id"] token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) @@ -368,17 +375,17 @@ class Auth(object): "User-Agent", default=[""] )[0] - if user and access_token and ip_addr: + if user_id and access_token and ip_addr: self.store.insert_client_ip( - user=user, + user=user_id, access_token=access_token, ip=ip_addr, user_agent=user_agent ) - request.authenticated_entity = user.to_string() + request.authenticated_entity = user_id.to_string() - defer.returnValue((user, token_id,)) + defer.returnValue((user_id, token_id,)) except KeyError: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", @@ -386,7 +393,7 @@ class Auth(object): ) @defer.inlineCallbacks - def get_user_by_access_token(self, token): + def _get_user_by_access_token(self, token): """ Get a registered user's ID. Args: @@ -396,6 +403,86 @@ class Auth(object): Raises: AuthError if no user by that token exists or the token is invalid. """ + try: + ret = yield self._get_user_from_macaroon(token) + except AuthError: + # TODO(daniel): Remove this fallback when all existing access tokens + # have been re-issued as macaroons. + ret = yield self._look_up_user_by_access_token(token) + defer.returnValue(ret) + + @defer.inlineCallbacks + def _get_user_from_macaroon(self, macaroon_str): + try: + macaroon = pymacaroons.Macaroon.deserialize(macaroon_str) + self._validate_macaroon(macaroon) + + user_prefix = "user_id = " + for caveat in macaroon.caveats: + if caveat.caveat_id.startswith(user_prefix): + user_id = UserID.from_string(caveat.caveat_id[len(user_prefix):]) + # This codepath exists so that we can actually return a + # token ID, because we use token IDs in place of device + # identifiers throughout the codebase. + # TODO(daniel): Remove this fallback when device IDs are + # properly implemented. + ret = yield self._look_up_user_by_access_token(macaroon_str) + if ret["user_id"] != user_id: + logger.error( + "Macaroon user (%s) != DB user (%s)", + user_id, + ret["user_id"] + ) + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, + "User mismatch in macaroon", + errcode=Codes.UNKNOWN_TOKEN + ) + defer.returnValue(ret) + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "No user caveat in macaroon", + errcode=Codes.UNKNOWN_TOKEN + ) + except (pymacaroons.exceptions.MacaroonException, TypeError, ValueError): + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Invalid macaroon passed.", + errcode=Codes.UNKNOWN_TOKEN + ) + + def _validate_macaroon(self, macaroon): + v = pymacaroons.Verifier() + v.satisfy_exact("gen = 1") + v.satisfy_exact("type = access") + v.satisfy_general(lambda c: c.startswith("user_id = ")) + v.satisfy_general(self._verify_expiry) + v.verify(macaroon, self.hs.config.macaroon_secret_key) + + v = pymacaroons.Verifier() + v.satisfy_general(self._verify_recognizes_caveats) + v.verify(macaroon, self.hs.config.macaroon_secret_key) + + def _verify_expiry(self, caveat): + prefix = "time < " + if not caveat.startswith(prefix): + return False + # TODO(daniel): Enable expiry check when clients actually know how to + # refresh tokens. (And remember to enable the tests) + return True + expiry = int(caveat[len(prefix):]) + now = self.hs.get_clock().time_msec() + return now < expiry + + def _verify_recognizes_caveats(self, caveat): + first_space = caveat.find(" ") + if first_space < 0: + return False + second_space = caveat.find(" ", first_space + 1) + if second_space < 0: + return False + return caveat[:second_space + 1] in self._KNOWN_CAVEAT_PREFIXES + + @defer.inlineCallbacks + def _look_up_user_by_access_token(self, token): ret = yield self.store.get_user_by_access_token(token) if not ret: raise AuthError( @@ -403,10 +490,9 @@ class Auth(object): errcode=Codes.UNKNOWN_TOKEN ) user_info = { - "user": UserID.from_string(ret.get("name")), + "user_id": UserID.from_string(ret.get("name")), "token_id": ret.get("token_id", None), } - defer.returnValue(user_info) @defer.inlineCallbacks diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 22fc804331..1ba85d6f83 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -14,22 +14,27 @@ # limitations under the License. from tests import unittest from twisted.internet import defer +from twisted.trial.unittest import FailTest from mock import Mock from synapse.api.auth import Auth from synapse.api.errors import AuthError +from synapse.types import UserID +from tests.utils import setup_test_homeserver + +import pymacaroons class AuthTestCase(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): self.state_handler = Mock() self.store = Mock() - self.hs = Mock() + self.hs = yield setup_test_homeserver(handlers=None) self.hs.get_datastore = Mock(return_value=self.store) - self.hs.get_state_handler = Mock(return_value=self.state_handler) self.auth = Auth(self.hs) self.test_user = "@foo:bar" @@ -133,3 +138,136 @@ class AuthTestCase(unittest.TestCase): request.requestHeaders.getRawHeaders = Mock(return_value=[""]) d = self.auth.get_user_by_req(request) self.failureResultOf(d, AuthError) + + @defer.inlineCallbacks + def test_get_user_from_macaroon(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + user_info = yield self.auth._get_user_from_macaroon(macaroon.serialize()) + user_id = user_info["user_id"] + self.assertEqual(UserID.from_string(user), user_id) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_user_db_mismatch(self): + self.store.get_user_by_access_token = Mock( + return_value={"name": "@percy:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("User mismatch", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_missing_caveat(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("No user caveat", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_wrong_key(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key + "wrong") + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("Invalid macaroon", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_unknown_caveat(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + macaroon.add_first_party_caveat("cunning > fox") + + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("Invalid macaroon", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_expired(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + self.todo = (FailTest, "Token expiry isn't currently enabled",) + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + macaroon.add_first_party_caveat("time < 1") # ms + + self.hs.clock.now = 5000 # seconds + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("Invalid macaroon", cm.exception.msg) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 91547bdd06..d8d1416f59 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -72,11 +72,11 @@ class PresenceStateTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(myid), + "user_id": UserID.from_string(myid), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token room_member_handler = hs.handlers.room_member_handler = Mock( spec=[ @@ -159,7 +159,7 @@ class PresenceListTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(myid), + "user_id": UserID.from_string(myid), "token_id": 1, } @@ -169,7 +169,7 @@ class PresenceListTestCase(unittest.TestCase): ] ) - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token presence.register_servlets(hs, self.mock_resource) diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 34ab47d02e..be1d52f720 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -56,10 +56,10 @@ class RoomPermissionsTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -441,10 +441,10 @@ class RoomsMemberListTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -519,10 +519,10 @@ class RoomsCreateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -610,11 +610,11 @@ class RoomTopicTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -715,10 +715,10 @@ class RoomMemberStateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -840,10 +840,10 @@ class RoomMessagesTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -935,10 +935,10 @@ class RoomInitialSyncTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 1c4519406d..da6fc975f7 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -63,11 +63,11 @@ class RoomTypingTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index c472d53043..85096a0326 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -37,9 +37,6 @@ class RestTestCase(unittest.TestCase): self.mock_resource = None self.auth_user_id = None - def mock_get_user_by_access_token(self, token=None): - return self.auth_user_id - @defer.inlineCallbacks def create_room_as(self, room_creator, is_public=True, tok=None): temp_id = self.auth_user_id diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index ef972a53aa..7d0f77a3ee 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -45,10 +45,10 @@ class V2AlphaRestTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.USER_ID), + "user_id": UserID.from_string(self.USER_ID), "token_id": 1, } - hs.get_auth().get_user_by_access_token = _get_user_by_access_token + hs.get_auth()._get_user_by_access_token = _get_user_by_access_token for r in self.TO_REGISTER: r.register_servlets(hs, self.mock_resource) -- cgit 1.5.1 From d3c0e488591386b7d24d23c6f6d3b237523fca89 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 26 Aug 2015 13:42:45 +0100 Subject: Merge erikj/user_dedup to develop --- synapse/handlers/auth.py | 39 +++++++++++++++++++++++++++++++-------- synapse/handlers/register.py | 4 ++-- synapse/rest/client/v1/login.py | 5 +++-- synapse/storage/registration.py | 14 ++++++++++++++ 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index c983d444e8..1ab19cd1a6 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -163,7 +163,8 @@ class AuthHandler(BaseHandler): if not user_id.startswith('@'): user_id = UserID.create(user_id, self.hs.hostname).to_string() - yield self._check_password(user_id, password) + user_id, password_hash = yield self._find_user_id_and_pwd_hash(user_id) + self._check_password(user_id, password, password_hash) defer.returnValue(user_id) @defer.inlineCallbacks @@ -280,27 +281,49 @@ class AuthHandler(BaseHandler): password (str): Password Returns: A tuple of: + The user's ID. The access token for the user's session. The refresh token for the user's session. Raises: StoreError if there was a problem storing the token. LoginError if there was an authentication problem. """ - yield self._check_password(user_id, password) + user_id, password_hash = yield self._find_user_id_and_pwd_hash(user_id) + self._check_password(user_id, password, password_hash) + logger.info("Logging in user %s", user_id) access_token = yield self.issue_access_token(user_id) refresh_token = yield self.issue_refresh_token(user_id) - defer.returnValue((access_token, refresh_token)) + defer.returnValue((user_id, access_token, refresh_token)) @defer.inlineCallbacks - def _check_password(self, user_id, password): - """Checks that user_id has passed password, raises LoginError if not.""" - user_info = yield self.store.get_user_by_id(user_id=user_id) - if not user_info: + def _find_user_id_and_pwd_hash(self, user_id): + """Checks to see if a user with the given id exists. Will check case + insensitively, but will throw if there are multiple inexact matches. + + Returns: + tuple: A 2-tuple of `(canonical_user_id, password_hash)` + """ + user_infos = yield self.store.get_users_by_id_case_insensitive(user_id) + if not user_infos: logger.warn("Attempted to login as %s but they do not exist", user_id) raise LoginError(403, "", errcode=Codes.FORBIDDEN) - stored_hash = user_info["password_hash"] + if len(user_infos) > 1: + if user_id not in user_infos: + logger.warn( + "Attempted to login as %s but it matches more than one user " + "inexactly: %r", + user_id, user_infos.keys() + ) + raise LoginError(403, "", errcode=Codes.FORBIDDEN) + + defer.returnValue((user_id, user_infos[user_id])) + else: + defer.returnValue(user_infos.popitem()) + + def _check_password(self, user_id, password, stored_hash): + """Checks that user_id has passed password, raises LoginError if not.""" if not bcrypt.checkpw(password, stored_hash): logger.warn("Failed password login for user %s", user_id) raise LoginError(403, "", errcode=Codes.FORBIDDEN) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 3d1b6531c2..56d125f753 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -56,8 +56,8 @@ class RegistrationHandler(BaseHandler): yield self.check_user_id_is_valid(user_id) - u = yield self.store.get_user_by_id(user_id) - if u: + users = yield self.store.get_users_by_id_case_insensitive(user_id) + if users: raise SynapseError( 400, "User ID already taken.", diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 3a0707c2ee..e580f71964 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -83,10 +83,11 @@ class LoginRestServlet(ClientV1RestServlet): if not user_id.startswith('@'): user_id = UserID.create( - user_id, self.hs.hostname).to_string() + user_id, self.hs.hostname + ).to_string() auth_handler = self.handlers.auth_handler - access_token, refresh_token = yield auth_handler.login_with_password( + user_id, access_token, refresh_token = yield auth_handler.login_with_password( user_id=user_id, password=login_submission["password"]) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a2d0f7c4b1..c9ceb132ae 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -120,6 +120,20 @@ class RegistrationStore(SQLBaseStore): allow_none=True, ) + def get_users_by_id_case_insensitive(self, user_id): + """Gets users that match user_id case insensitively. + Returns a mapping of user_id -> password_hash. + """ + def f(txn): + sql = ( + "SELECT name, password_hash FROM users" + " WHERE lower(name) = lower(?)" + ) + txn.execute(sql, (user_id,)) + return dict(txn.fetchall()) + + return self.runInteraction("get_users_by_id_case_insensitive", f) + @defer.inlineCallbacks def user_set_password_hash(self, user_id, password_hash): """ -- cgit 1.5.1 From 81450fded8c4d2a0f4a914251cc2d11a366efdbd Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 26 Aug 2015 13:56:01 +0100 Subject: Turn TODO into thing which actually will fail --- tests/api/test_auth.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 1ba85d6f83..2e2d0c428a 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -14,7 +14,6 @@ # limitations under the License. from tests import unittest from twisted.internet import defer -from twisted.trial.unittest import FailTest from mock import Mock @@ -251,7 +250,6 @@ class AuthTestCase(unittest.TestCase): return_value={"name": "@baldrick:matrix.org"} ) - self.todo = (FailTest, "Token expiry isn't currently enabled",) self.store.get_user_by_access_token = Mock( return_value={"name": "@baldrick:matrix.org"} ) @@ -267,7 +265,12 @@ class AuthTestCase(unittest.TestCase): macaroon.add_first_party_caveat("time < 1") # ms self.hs.clock.now = 5000 # seconds - with self.assertRaises(AuthError) as cm: - yield self.auth._get_user_from_macaroon(macaroon.serialize()) - self.assertEqual(401, cm.exception.code) - self.assertIn("Invalid macaroon", cm.exception.msg) + + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + # TODO(daniel): Turn on the check that we validate expiration, when we + # validate expiration (and remove the above line, which will start + # throwing). + # with self.assertRaises(AuthError) as cm: + # yield self.auth._get_user_from_macaroon(macaroon.serialize()) + # self.assertEqual(401, cm.exception.code) + # self.assertIn("Invalid macaroon", cm.exception.msg) -- cgit 1.5.1 From 3063383547529a542b48f416d64fd98eaf6a2f60 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 26 Aug 2015 15:59:32 +0100 Subject: Swap out bcrypt for md5 in tests This reduces our ~8 second sequential test time down to ~7 seconds --- synapse/handlers/auth.py | 27 +++++++++++++++++++++++++-- synapse/handlers/register.py | 2 +- tests/utils.py | 13 +++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1ab19cd1a6..59f687e0f1 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -324,7 +324,7 @@ class AuthHandler(BaseHandler): def _check_password(self, user_id, password, stored_hash): """Checks that user_id has passed password, raises LoginError if not.""" - if not bcrypt.checkpw(password, stored_hash): + if not self.validate_hash(password, stored_hash): logger.warn("Failed password login for user %s", user_id) raise LoginError(403, "", errcode=Codes.FORBIDDEN) @@ -369,7 +369,7 @@ class AuthHandler(BaseHandler): @defer.inlineCallbacks def set_password(self, user_id, newpassword): - password_hash = bcrypt.hashpw(newpassword, bcrypt.gensalt()) + password_hash = self.hash(newpassword) yield self.store.user_set_password_hash(user_id, password_hash) yield self.store.user_delete_access_tokens(user_id) @@ -391,3 +391,26 @@ class AuthHandler(BaseHandler): def _remove_session(self, session): logger.debug("Removing session %s", session) del self.sessions[session["id"]] + + def hash(self, password): + """Computes a secure hash of password. + + Args: + password (str): Password to hash. + + Returns: + Hashed password (str). + """ + return bcrypt.hashpw(password, bcrypt.gensalt()) + + def validate_hash(self, password, stored_hash): + """Validates that self.hash(password) == stored_hash. + + Args: + password (str): Password to hash. + stored_hash (str): Expected hash value. + + Returns: + Whether self.hash(password) == stored_hash (bool). + """ + return bcrypt.checkpw(password, stored_hash) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 56d125f753..855bb58522 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -82,7 +82,7 @@ class RegistrationHandler(BaseHandler): yield run_on_reactor() password_hash = None if password: - password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) + password_hash = self.auth_handler().hash(password) if localpart: yield self.check_username(localpart) diff --git a/tests/utils.py b/tests/utils.py index 3766a994f2..dd19a16fc7 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -27,6 +27,7 @@ from twisted.enterprise.adbapi import ConnectionPool from collections import namedtuple from mock import patch, Mock +import hashlib import urllib import urlparse @@ -67,6 +68,18 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): **kargs ) + # bcrypt is far too slow to be doing in unit tests + def swap_out_hash_for_testing(old_build_handlers): + def build_handlers(): + handlers = old_build_handlers() + auth_handler = handlers.auth_handler + auth_handler.hash = lambda p: hashlib.md5(p).hexdigest() + auth_handler.validate_hash = lambda p, h: hashlib.md5(p).hexdigest() == h + return handlers + return build_handlers + + hs.build_handlers = swap_out_hash_for_testing(hs.build_handlers) + defer.returnValue(hs) -- cgit 1.5.1 From 86fac9c95ebb034f5e0ad9a4ffdf868599727e49 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 26 Aug 2015 16:03:17 +0100 Subject: Remove unused import --- synapse/handlers/register.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 855bb58522..ef4081e3fe 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -25,7 +25,6 @@ import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor from synapse.http.client import CaptchaServerHttpClient -import bcrypt import logging import urllib -- cgit 1.5.1 From 8256a8ece7e228bf69fcd352f1b4adfa2138719a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 28 Aug 2015 15:31:49 +0100 Subject: Allow users to redact their own events --- synapse/api/auth.py | 33 +++++++++++++++++++++++++-------- synapse/handlers/_base.py | 19 +++++++++++++++++-- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 65ee1452ce..f63d2daad8 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -20,7 +20,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError from synapse.util.logutils import log_function -from synapse.types import UserID +from synapse.types import UserID, EventID import logging @@ -91,7 +91,7 @@ class Auth(object): self._check_power_levels(event, auth_events) if event.type == EventTypes.Redaction: - self._check_redaction(event, auth_events) + self.check_redaction(event, auth_events) logger.debug("Allowing! %s", event) except AuthError as e: @@ -541,16 +541,33 @@ class Auth(object): return True - def _check_redaction(self, event, auth_events): + def check_redaction(self, event, auth_events): + """Check whether the event sender is allowed to redact the target event. + + Returns: + True if the the sender is allowed to redact the target event if the + target event was created by them. + False if the sender is allowed to redact the target event with no + further checks. + + Raises: + AuthError if the event sender is definitely not allowed to redact + the target event. + """ user_level = self._get_user_power_level(event.user_id, auth_events) redact_level = self._get_named_level(auth_events, "redact", 50) - if user_level < redact_level: - raise AuthError( - 403, - "You don't have permission to redact events" - ) + if user_level > redact_level: + return False + + if EventID.from_string(event.redacts).domain == self.hs.get_config().server_name: + return True + + raise AuthError( + 403, + "You don't have permission to redact events" + ) def _check_power_levels(self, event, auth_events): user_list = event.content.get("users", {}) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index e91f1129db..9d36e3b6d2 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.errors import LimitExceededError, SynapseError +from synapse.api.errors import LimitExceededError, SynapseError, AuthError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes from synapse.types import UserID, RoomAlias @@ -131,7 +131,7 @@ class BaseHandler(object): ) if event.type == EventTypes.CanonicalAlias: - # Check the alias is acually valid (at this time at least) + # Check the alias is actually valid (at this time at least) room_alias_str = event.content.get("alias", None) if room_alias_str: room_alias = RoomAlias.from_string(room_alias_str) @@ -146,6 +146,21 @@ class BaseHandler(object): ) ) + if event.type == EventTypes.Redaction: + if self.auth.check_redaction(event, auth_events=context.current_state): + original_event = yield self.store.get_event( + event.redacts, + check_redacted=False, + get_prev_content=False, + allow_rejected=False, + allow_none=False + ) + if event.user_id != original_event.user_id: + raise AuthError( + 403, + "You don't have permission to redact events" + ) + destinations = set(extra_destinations) for k, s in context.current_state.items(): try: -- cgit 1.5.1 From d71af2ee122cceaa100e17575fea9dd830c6ce3e Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 29 Aug 2015 22:23:21 +0100 Subject: don't log the whole DB config (including postgres password...) --- synapse/app/homeserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ff7807c2e6..ffc6299146 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -403,7 +403,7 @@ def setup(config_options): database_engine=database_engine, ) - logger.info("Preparing database: %r...", config.database_config) + logger.info("Preparing database: %s...", config.database_config['name']) try: db_conn = database_engine.module.connect( @@ -425,7 +425,7 @@ def setup(config_options): ) sys.exit(1) - logger.info("Database prepared in %r.", config.database_config) + logger.info("Database prepared in %s.", config.database_config['name']) hs.start_listening() -- cgit 1.5.1 From b854a375b0a859f8c4e16adf9f2fcec75b5a6816 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Sep 2015 11:53:31 +0100 Subject: Check domain of events properly Federated servers still need to delegate authority to owning servers --- synapse/api/auth.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f63d2daad8..0c0d678562 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -561,7 +561,9 @@ class Auth(object): if user_level > redact_level: return False - if EventID.from_string(event.redacts).domain == self.hs.get_config().server_name: + redacter_domain = EventID.from_string(event.event_id).domain + redactee_domain = EventID.from_string(event.redacts).domain + if redacter_domain == redactee_domain: return True raise AuthError( -- cgit 1.5.1 From e255c2c32ff85db03abbf2dac184b2949f481cfb Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Sep 2015 12:41:16 +0100 Subject: s/user_id/user/g for consistency --- synapse/api/auth.py | 20 ++++++++++---------- tests/api/test_auth.py | 8 ++++---- tests/rest/client/v1/test_presence.py | 4 ++-- tests/rest/client/v1/test_rooms.py | 14 +++++++------- tests/rest/client/v1/test_typing.py | 2 +- tests/rest/client/v2_alpha/__init__.py | 2 +- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f8ea1e2c69..0a77a76cb8 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -367,7 +367,7 @@ class Auth(object): pass # normal users won't have the user_id query parameter set. user_info = yield self._get_user_by_access_token(access_token) - user_id = user_info["user_id"] + user = user_info["user"] token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) @@ -375,17 +375,17 @@ class Auth(object): "User-Agent", default=[""] )[0] - if user_id and access_token and ip_addr: + if user and access_token and ip_addr: self.store.insert_client_ip( - user=user_id, + user=user, access_token=access_token, ip=ip_addr, user_agent=user_agent ) - request.authenticated_entity = user_id.to_string() + request.authenticated_entity = user.to_string() - defer.returnValue((user_id, token_id,)) + defer.returnValue((user, token_id,)) except KeyError: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", @@ -420,18 +420,18 @@ class Auth(object): user_prefix = "user_id = " for caveat in macaroon.caveats: if caveat.caveat_id.startswith(user_prefix): - user_id = UserID.from_string(caveat.caveat_id[len(user_prefix):]) + user = UserID.from_string(caveat.caveat_id[len(user_prefix):]) # This codepath exists so that we can actually return a # token ID, because we use token IDs in place of device # identifiers throughout the codebase. # TODO(daniel): Remove this fallback when device IDs are # properly implemented. ret = yield self._look_up_user_by_access_token(macaroon_str) - if ret["user_id"] != user_id: + if ret["user"] != user: logger.error( "Macaroon user (%s) != DB user (%s)", - user_id, - ret["user_id"] + user, + ret["user"] ) raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, @@ -490,7 +490,7 @@ class Auth(object): errcode=Codes.UNKNOWN_TOKEN ) user_info = { - "user_id": UserID.from_string(ret.get("name")), + "user": UserID.from_string(ret.get("name")), "token_id": ret.get("token_id", None), } defer.returnValue(user_info) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 2e2d0c428a..c96273480d 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -146,17 +146,17 @@ class AuthTestCase(unittest.TestCase): return_value={"name": "@baldrick:matrix.org"} ) - user = "@baldrick:matrix.org" + user_id = "@baldrick:matrix.org" macaroon = pymacaroons.Macaroon( location=self.hs.config.server_name, identifier="key", key=self.hs.config.macaroon_secret_key) macaroon.add_first_party_caveat("gen = 1") macaroon.add_first_party_caveat("type = access") - macaroon.add_first_party_caveat("user_id = %s" % (user,)) + macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) user_info = yield self.auth._get_user_from_macaroon(macaroon.serialize()) - user_id = user_info["user_id"] - self.assertEqual(UserID.from_string(user), user_id) + user = user_info["user"] + self.assertEqual(UserID.from_string(user_id), user) @defer.inlineCallbacks def test_get_user_from_macaroon_user_db_mismatch(self): diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index d8d1416f59..2ee3da0b34 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -72,7 +72,7 @@ class PresenceStateTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(myid), + "user": UserID.from_string(myid), "token_id": 1, } @@ -159,7 +159,7 @@ class PresenceListTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(myid), + "user": UserID.from_string(myid), "token_id": 1, } diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index be1d52f720..9fb2bfb315 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -56,7 +56,7 @@ class RoomPermissionsTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -441,7 +441,7 @@ class RoomsMemberListTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -519,7 +519,7 @@ class RoomsCreateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -610,7 +610,7 @@ class RoomTopicTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } @@ -715,7 +715,7 @@ class RoomMemberStateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -840,7 +840,7 @@ class RoomMessagesTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -935,7 +935,7 @@ class RoomInitialSyncTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index da6fc975f7..6395ce79db 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -63,7 +63,7 @@ class RoomTypingTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index 7d0f77a3ee..f45570a1c0 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -45,7 +45,7 @@ class V2AlphaRestTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.USER_ID), + "user": UserID.from_string(self.USER_ID), "token_id": 1, } hs.get_auth()._get_user_by_access_token = _get_user_by_access_token -- cgit 1.5.1 From 57df6fffa7c3dd71f07012f62c4d63909adcade7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 1 Sep 2015 14:47:57 +0100 Subject: Use the version of "matrix-angular-sdk" hosted on pypi --- synapse/python_dependencies.py | 5 ----- tox.ini | 2 -- 2 files changed, 7 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 8ec272fd5f..795ef27182 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -56,11 +56,6 @@ def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) DEPENDENCY_LINKS = { - "matrix-angular-sdk": github_link( - project="matrix-org/matrix-angular-sdk", - version="v0.6.6", - egg="matrix_angular_sdk-0.6.6", - ), } diff --git a/tox.ini b/tox.ini index 3f0649b628..a69948484f 100644 --- a/tox.ini +++ b/tox.ini @@ -11,8 +11,6 @@ setenv = commands = coverage run --source=synapse {envbindir}/trial {posargs:tests} coverage report -m -install_command = - pip install --process-dependency-links --pre {opts} {packages} [testenv:packaging] deps = -- cgit 1.5.1 From ab9e01809def16dca19d6391ab1234eaa423b182 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Sep 2015 15:15:19 +0100 Subject: Check room exists when authenticating an event, by asserting they reference a creation event --- synapse/api/auth.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 65ee1452ce..410f4c11e7 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -65,6 +65,14 @@ class Auth(object): # FIXME return True + creation_event = auth_events.get((EventTypes.Create, ""), None) + + if not creation_event: + raise SynapseError( + 400, + "Room %r does not exist" % (event.room_id,) + ) + # FIXME: Temp hack if event.type == EventTypes.Aliases: return True -- cgit 1.5.1 From 00149c063b8f81548bd3eefd3e497acc03512d35 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Sep 2015 15:42:03 +0100 Subject: Fix tests --- synapse/api/auth.py | 2 +- tests/test_state.py | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 410f4c11e7..df7fb6aab7 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -69,7 +69,7 @@ class Auth(object): if not creation_event: raise SynapseError( - 400, + 403, "Room %r does not exist" % (event.room_id,) ) diff --git a/tests/test_state.py b/tests/test_state.py index 5845358754..55f37c521f 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -204,8 +204,8 @@ class StateTestCase(unittest.TestCase): nodes={ "START": DictObj( type=EventTypes.Create, - state_key="creator", - content={"membership": "@user_id:example.com"}, + state_key="", + content={"creator": "@user_id:example.com"}, depth=1, ), "A": DictObj( @@ -259,8 +259,8 @@ class StateTestCase(unittest.TestCase): nodes={ "START": DictObj( type=EventTypes.Create, - state_key="creator", - content={"membership": "@user_id:example.com"}, + state_key="", + content={"creator": "@user_id:example.com"}, depth=1, ), "A": DictObj( @@ -432,13 +432,19 @@ class StateTestCase(unittest.TestCase): def test_resolve_message_conflict(self): event = create_event(type="test_message", name="event") + creation = create_event( + type=EventTypes.Create, state_key="" + ) + old_state_1 = [ + creation, create_event(type="test1", state_key="1"), create_event(type="test1", state_key="2"), create_event(type="test2", state_key=""), ] old_state_2 = [ + creation, create_event(type="test1", state_key="1"), create_event(type="test3", state_key="2"), create_event(type="test4", state_key=""), @@ -446,7 +452,7 @@ class StateTestCase(unittest.TestCase): context = yield self._get_context(event, old_state_1, old_state_2) - self.assertEqual(len(context.current_state), 5) + self.assertEqual(len(context.current_state), 6) self.assertIsNone(context.state_group) @@ -454,13 +460,19 @@ class StateTestCase(unittest.TestCase): def test_resolve_state_conflict(self): event = create_event(type="test4", state_key="", name="event") + creation = create_event( + type=EventTypes.Create, state_key="" + ) + old_state_1 = [ + creation, create_event(type="test1", state_key="1"), create_event(type="test1", state_key="2"), create_event(type="test2", state_key=""), ] old_state_2 = [ + creation, create_event(type="test1", state_key="1"), create_event(type="test3", state_key="2"), create_event(type="test4", state_key=""), @@ -468,7 +480,7 @@ class StateTestCase(unittest.TestCase): context = yield self._get_context(event, old_state_1, old_state_2) - self.assertEqual(len(context.current_state), 5) + self.assertEqual(len(context.current_state), 6) self.assertIsNone(context.state_group) @@ -484,36 +496,45 @@ class StateTestCase(unittest.TestCase): } ) + creation = create_event( + type=EventTypes.Create, state_key="", + content={"creator": "@foo:bar"} + ) + old_state_1 = [ + creation, member_event, create_event(type="test1", state_key="1", depth=1), ] old_state_2 = [ + creation, member_event, create_event(type="test1", state_key="1", depth=2), ] context = yield self._get_context(event, old_state_1, old_state_2) - self.assertEqual(old_state_2[1], context.current_state[("test1", "1")]) + self.assertEqual(old_state_2[2], context.current_state[("test1", "1")]) # Reverse the depth to make sure we are actually using the depths # during state resolution. old_state_1 = [ + creation, member_event, create_event(type="test1", state_key="1", depth=2), ] old_state_2 = [ + creation, member_event, create_event(type="test1", state_key="1", depth=1), ] context = yield self._get_context(event, old_state_1, old_state_2) - self.assertEqual(old_state_1[1], context.current_state[("test1", "1")]) + self.assertEqual(old_state_1[2], context.current_state[("test1", "1")]) def _get_context(self, event, old_state_1, old_state_2): group_name_1 = "group_name_1" -- cgit 1.5.1 From a88e16152f00719df152eaef31dcfd457c019293 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Sep 2015 15:09:23 +0100 Subject: Add flag which disables federation of the room --- synapse/api/auth.py | 16 +++++++++++++++- synapse/handlers/room.py | 8 ++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 65ee1452ce..f7cf17e433 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -20,7 +20,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError from synapse.util.logutils import log_function -from synapse.types import UserID +from synapse.types import EventID, RoomID, UserID import logging @@ -65,6 +65,15 @@ class Auth(object): # FIXME return True + creating_domain = RoomID.from_string(event.room_id).domain + originating_domain = EventID.from_string(event.event_id).domain + if creating_domain != originating_domain: + if not self.can_federate(event, auth_events): + raise SynapseError( + 403, + "This room has been marked as unfederatable." + ) + # FIXME: Temp hack if event.type == EventTypes.Aliases: return True @@ -153,6 +162,11 @@ class Auth(object): user_id, room_id, repr(member) )) + def can_federate(self, event, auth_events): + creation_event = auth_events.get((EventTypes.Create, "")) + + return creation_event.content.get("m.federate", True) is True + @log_function def is_membership_change_allowed(self, event, auth_events): membership = event.content["membership"] diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c5d1001b50..4f8ad824b5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -150,12 +150,15 @@ class RoomCreationHandler(BaseHandler): for val in raw_initial_state: initial_state[(val["type"], val.get("state_key", ""))] = val["content"] + creation_content = config.get("creation_content", {}) + user = UserID.from_string(user_id) creation_events = self._create_events_for_new_room( user, room_id, preset_config=preset_config, invite_list=invite_list, initial_state=initial_state, + creation_content=creation_content, ) msg_handler = self.hs.get_handlers().message_handler @@ -203,7 +206,7 @@ class RoomCreationHandler(BaseHandler): defer.returnValue(result) def _create_events_for_new_room(self, creator, room_id, preset_config, - invite_list, initial_state): + invite_list, initial_state, creation_content): config = RoomCreationHandler.PRESETS_DICT[preset_config] creator_id = creator.to_string() @@ -225,9 +228,10 @@ class RoomCreationHandler(BaseHandler): return e + creation_content.update({"creator": creator.to_string()}) creation_event = create( etype=EventTypes.Create, - content={"creator": creator.to_string()}, + content=creation_content, ) join_event = create( -- cgit 1.5.1 From 7ab401d4dc353c43217ef5656347d68fe7d2ffa4 Mon Sep 17 00:00:00 2001 From: pztrn Date: Tue, 1 Sep 2015 19:48:22 +0500 Subject: Ignore development virtualenv and generated logger configuration as well. Signed-off-by: Stanislav Nikitin --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4c336b710d..f8c4000134 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ build/ localhost-800*/ static/client/register/register_config.js .tox + +env/ +*.config -- cgit 1.5.1 From b345853918b9300bdde19010d29bf66973497de7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Sep 2015 15:57:35 +0100 Subject: Check against sender rather than event_id --- synapse/api/auth.py | 6 +++--- tests/test_state.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f7cf17e433..75b7c467b5 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -20,7 +20,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError from synapse.util.logutils import log_function -from synapse.types import EventID, RoomID, UserID +from synapse.types import RoomID, UserID import logging @@ -66,10 +66,10 @@ class Auth(object): return True creating_domain = RoomID.from_string(event.room_id).domain - originating_domain = EventID.from_string(event.event_id).domain + originating_domain = UserID.from_string(event.sender).domain if creating_domain != originating_domain: if not self.can_federate(event, auth_events): - raise SynapseError( + raise AuthError( 403, "This room has been marked as unfederatable." ) diff --git a/tests/test_state.py b/tests/test_state.py index 5845358754..04c4439183 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -35,7 +35,7 @@ def create_event(name=None, type=None, state_key=None, depth=2, event_id=None, if not event_id: _next_event_id += 1 - event_id = str(_next_event_id) + event_id = "$%s:test" % (_next_event_id,) if not name: if state_key is not None: -- cgit 1.5.1 From 9b05ef6f394bae9c844ead1a5edf53d1ef6c4fd7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Sep 2015 16:17:25 +0100 Subject: Also check the domains for membership state_keys --- synapse/api/auth.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 90f11fdc94..944fbbf532 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -190,6 +190,15 @@ class Auth(object): target_user_id = event.state_key + creating_domain = RoomID.from_string(event.room_id).domain + target_domain = UserID.from_string(target_user_id).domain + if creating_domain != target_domain: + if not self.can_federate(event, auth_events): + raise AuthError( + 403, + "This room has been marked as unfederatable." + ) + # get info about the caller key = (EventTypes.Member, event.user_id, ) caller = auth_events.get(key) -- cgit 1.5.1 From 3cdfd37d95484b9ebca012a8169b8d6613385656 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 1 Sep 2015 16:47:26 +0100 Subject: Print an example "pip install" line for a missing requirement --- synapse/app/homeserver.py | 17 +++++++++++++++-- synapse/python_dependencies.py | 17 +++++++++++------ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ffc6299146..2a85932b86 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -16,10 +16,23 @@ import sys sys.dont_write_bytecode = True -from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS +from synapse.python_dependencies import ( + check_requirements, DEPENDENCY_LINKS, MissingRequirementError +) if __name__ == '__main__': - check_requirements() + try: + check_requirements() + except MissingRequirementError as e: + message = "\n".join([ + "Missing Requirement: %s" % (e.message,), + "To install run:", + " pip install --upgrade --force \"%s\"" % (e.dependency,), + "", + ]) + sys.stderr.writelines(message) + sys.exit(1) + from synapse.storage.engines import create_engine, IncorrectDatabaseSetup from synapse.storage import ( diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 795ef27182..387183b50b 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -60,7 +60,10 @@ DEPENDENCY_LINKS = { class MissingRequirementError(Exception): - pass + def __init__(self, message, module_name, dependency): + super(MissingRequirementError, self).__init__(message) + self.module_name = module_name + self.dependency = dependency def check_requirements(config=None): @@ -88,7 +91,7 @@ def check_requirements(config=None): ) raise MissingRequirementError( "Can't import %r which is part of %r" - % (module_name, dependency) + % (module_name, dependency), module_name, dependency ) version = getattr(module, "__version__", None) file_path = getattr(module, "__file__", None) @@ -101,23 +104,25 @@ def check_requirements(config=None): if version is None: raise MissingRequirementError( "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name) + % (dependency, module_name), module_name, dependency ) if LooseVersion(version) < LooseVersion(required_version): raise MissingRequirementError( "Version of %r in %r is too old. %r < %r" - % (dependency, file_path, version, required_version) + % (dependency, file_path, version, required_version), + module_name, dependency ) elif version_test == "==": if version is None: raise MissingRequirementError( "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name) + % (dependency, module_name), module_name, dependency ) if LooseVersion(version) != LooseVersion(required_version): raise MissingRequirementError( "Unexpected version of %r in %r. %r != %r" - % (dependency, file_path, version, required_version) + % (dependency, file_path, version, required_version), + module_name, dependency ) -- cgit 1.5.1 From 8bab7abddda956b4131d8e4ad04b623c93df02ad Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 1 Sep 2015 16:51:10 +0100 Subject: Add nacl.bindings to the list of modules checked. Re-arrange import order to check packages after the packages they depend on --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 387183b50b..e95316720e 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -18,18 +18,18 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { + "frozendict>=0.4": ["frozendict"], "unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"], "canonicaljson>=1.0.0": ["canonicaljson>=1.0.0"], "signedjson>=1.0.0": ["signedjson>=1.0.0"], - "Twisted>=15.1.0": ["twisted>=15.1.0"], + "pynacl>=0.3.0": ["nacl>=0.3.0", "nacl.bindings"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], + "Twisted>=15.1.0": ["twisted>=15.1.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], "pyasn1": ["pyasn1"], - "pynacl>=0.3.0": ["nacl>=0.3.0"], "daemonize": ["daemonize"], "py-bcrypt": ["bcrypt"], - "frozendict>=0.4": ["frozendict"], "pillow": ["PIL"], "pydenticon": ["pydenticon"], "ujson": ["ujson"], -- cgit 1.5.1 From bdf2e5865a6c47b89dbcb11fb747cfc87c26fdbd Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 3 Sep 2015 09:51:42 +0300 Subject: update logger to match new ambiguous script name... --- scripts/synapse_port_db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index e7ed4c309b..c241fdec5a 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -29,7 +29,7 @@ import traceback import yaml -logger = logging.getLogger("port_from_sqlite_to_postgres") +logger = logging.getLogger("synapse_port_db") BOOLEAN_COLUMNS = { -- cgit 1.5.1 From 9e4dacd5e7e560bae614ddf390a9f02c2c19a25d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 7 Sep 2015 16:45:48 +0100 Subject: The maxrss reported by getrusage is in kilobytes, not pages --- synapse/metrics/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index d7bcad8a8a..3f7b7158aa 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -17,7 +17,7 @@ from __future__ import absolute_import import logging -from resource import getrusage, getpagesize, RUSAGE_SELF +from resource import getrusage, RUSAGE_SELF import functools import os import stat @@ -100,7 +100,6 @@ def render_all(): # process resource usage rusage = None -PAGE_SIZE = getpagesize() def update_resource_metrics(): @@ -113,8 +112,8 @@ resource_metrics = get_metrics_for("process.resource") resource_metrics.register_callback("utime", lambda: rusage.ru_utime * 1000) resource_metrics.register_callback("stime", lambda: rusage.ru_stime * 1000) -# pages -resource_metrics.register_callback("maxrss", lambda: rusage.ru_maxrss * PAGE_SIZE) +# kilobytes +resource_metrics.register_callback("maxrss", lambda: rusage.ru_maxrss * 1024) TYPES = { stat.S_IFSOCK: "SOCK", -- cgit 1.5.1 From 709ba99afda371bb19a34e4448dc771a6cdab320 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 7 Sep 2015 16:45:55 +0100 Subject: Check that /proc/self/fd exists before listing it --- synapse/metrics/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 3f7b7158aa..943d637459 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -130,6 +130,10 @@ def _process_fds(): counts = {(k,): 0 for k in TYPES.values()} counts[("other",)] = 0 + # Not every OS will have a /proc/self/fd directory + if not os.path.exists("/proc/self/fd"): + return counts + for fd in os.listdir("/proc/self/fd"): try: s = os.stat("/proc/self/fd/%s" % (fd)) -- cgit 1.5.1 From 417485eefaff86206b5f961102f882b3fbe44651 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 8 Sep 2015 18:14:54 +0100 Subject: Include the event_id and stream_ordering of membership events when looking up which rooms a user is in --- synapse/storage/roommember.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 8eee2dfbcc..cd9eefbd9f 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) RoomsForUser = namedtuple( "RoomsForUser", - ("room_id", "sender", "membership") + ("room_id", "sender", "membership", "event_id", "stream_ordering") ) @@ -141,9 +141,11 @@ class RoomMemberStore(SQLBaseStore): args.extend(membership_list) sql = ( - "SELECT m.room_id, m.sender, m.membership" + "SELECT m.room_id, m.sender, m.membership, m.event_id, e.stream_ordering" " FROM room_memberships as m" " INNER JOIN current_state_events as c" + " ON e.event_id = c.event_id " + " INNER JOIN events as e " " ON m.event_id = c.event_id " " AND m.room_id = c.room_id " " AND m.user_id = c.state_key" -- cgit 1.5.1 From dd42bb78d088c719aaa288b892fe58caa5850deb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 8 Sep 2015 18:16:09 +0100 Subject: Include rooms that a user has left in an initialSync. Include the state and messages at the point they left the room --- synapse/handlers/message.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 23b779ad7c..5447c97e83 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -316,7 +316,9 @@ class MessageHandler(BaseHandler): """ room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=user_id, - membership_list=[Membership.INVITE, Membership.JOIN] + membership_list=[ + Membership.INVITE, Membership.JOIN, Membership.LEAVE + ] ) user = UserID.from_string(user_id) @@ -358,19 +360,32 @@ class MessageHandler(BaseHandler): rooms_ret.append(d) - if event.membership != Membership.JOIN: + if event.membership not in (Membership.JOIN, Membership.LEAVE): return try: + if event.membership == Membership.JOIN: + room_end_token = now_token.room_key + deferred_room_state = self.state_handler.get_current_state( + event.room_id + ) + else: + room_end_token = "s%d" % (event.stream_ordering,) + deferred_room_state = self.store.get_state_for_events( + event.room_id, [event.event_id], None + ) + deferred_room_state.addCallback( + lambda states: states[event.event_id] + ) + + (messages, token), current_state = yield defer.gatherResults( [ self.store.get_recent_events_for_room( event.room_id, limit=limit, - end_token=now_token.room_key, - ), - self.state_handler.get_current_state( - event.room_id + end_token=room_end_token, ), + deferred_room_state, ] ).addErrback(unwrapFirstError) -- cgit 1.5.1 From e530208e68a2d0c37732855e4ed8f22a9f84d334 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Sep 2015 09:57:49 +0100 Subject: Change default history visibility for private rooms --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c5d1001b50..bb3d428288 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -39,7 +39,7 @@ class RoomCreationHandler(BaseHandler): PRESETS_DICT = { RoomCreationPreset.PRIVATE_CHAT: { "join_rules": JoinRules.INVITE, - "history_visibility": "invited", + "history_visibility": "shared", "original_invitees_have_ops": False, }, RoomCreationPreset.PUBLIC_CHAT: { -- cgit 1.5.1 From 81a93ddcc8798568276582ed9c7a63bc64dc5bc0 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 12:02:07 +0100 Subject: Allow configuration to ignore invalid SSL certs This will be useful for sytest, and sytest only, hence the aggressive config key name. --- synapse/app/homeserver.py | 8 ++++---- synapse/config/tls.py | 4 ++++ synapse/crypto/keyring.py | 4 ++-- synapse/handlers/auth.py | 3 +-- synapse/http/client.py | 25 +++++++++++++++++++++++-- synapse/http/matrixfederationclient.py | 4 ++-- synapse/server.py | 14 ++++++++++++++ 7 files changed, 50 insertions(+), 12 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ffc6299146..ba76ee362a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -15,6 +15,7 @@ # limitations under the License. import sys + sys.dont_write_bytecode = True from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS @@ -221,7 +222,7 @@ class SynapseHomeServer(HomeServer): listener_config, root_resource, ), - self.tls_context_factory, + self.tls_server_context_factory, interface=bind_address ) else: @@ -365,7 +366,6 @@ def setup(config_options): Args: config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. - should_run (bool): Whether to start the reactor. Returns: HomeServer @@ -388,7 +388,7 @@ def setup(config_options): events.USE_FROZEN_DICTS = config.use_frozen_dicts - tls_context_factory = context_factory.ServerContextFactory(config) + tls_server_context_factory = context_factory.ServerContextFactory(config) database_engine = create_engine(config.database_config["name"]) config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection @@ -396,7 +396,7 @@ def setup(config_options): hs = SynapseHomeServer( config.server_name, db_config=config.database_config, - tls_context_factory=tls_context_factory, + tls_server_context_factory=tls_server_context_factory, config=config, content_addr=config.content_addr, version_string=version_string, diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 4751d39bc9..472cf7ac4a 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -42,6 +42,10 @@ class TlsConfig(Config): config.get("tls_dh_params_path"), "tls_dh_params" ) + self.use_insecure_ssl_client = config.get( + "i_really_want_to_ignore_ssl_certs_when_i_am_an_http_client_even_" + "though_it_is_woefully_insecure_because_i_hate_my_users", False) + def default_config(self, config_dir_path, server_name): base_key_name = os.path.join(config_dir_path, server_name) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index a692cdbe55..e98a625fea 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -463,7 +463,7 @@ class Keyring(object): continue (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory, + server_name, self.hs.tls_server_context_factory, path=(b"/_matrix/key/v2/server/%s" % ( urllib.quote(requested_key_id), )).encode("ascii"), @@ -597,7 +597,7 @@ class Keyring(object): # Try to fetch the key from the remote server. (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory + server_name, self.hs.tls_server_context_factory ) # Check the response. diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 59f687e0f1..793b3fcd8b 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -19,7 +19,6 @@ from ._base import BaseHandler from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes -from synapse.http.client import SimpleHttpClient from synapse.util.async import run_on_reactor from twisted.web.client import PartialDownloadError @@ -187,7 +186,7 @@ class AuthHandler(BaseHandler): # TODO: get this from the homeserver rather than creating a new one for # each request try: - client = SimpleHttpClient(self.hs) + client = self.hs.get_simple_http_client() resp_body = yield client.post_urlencoded_get_json( self.hs.config.recaptcha_siteverify_api, args={ diff --git a/synapse/http/client.py b/synapse/http/client.py index 4b8fd3d3a3..da77c8b0ac 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -12,6 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from OpenSSL import SSL +from OpenSSL.SSL import VERIFY_NONE from synapse.api.errors import CodeMessageException from synapse.util.logcontext import preserve_context_over_fn @@ -19,7 +21,7 @@ import synapse.metrics from canonicaljson import encode_canonical_json -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, ssl from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError, HTTPConnectionPool, @@ -59,7 +61,12 @@ class SimpleHttpClient(object): # 'like a browser' pool = HTTPConnectionPool(reactor) pool.maxPersistentPerHost = 10 - self.agent = Agent(reactor, pool=pool) + self.agent = Agent( + reactor, + pool=pool, + connectTimeout=15, + contextFactory=hs.get_http_client_context_factory() + ) self.version_string = hs.version_string def request(self, method, uri, *args, **kwargs): @@ -252,3 +259,17 @@ def _print_ex(e): _print_ex(ex) else: logger.exception(e) + + +class WoefullyInsecureContextFactory(ssl.ContextFactory): + """ + Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. + + Do not use this unless you really, really hate your users.""" + + def __init__(self): + self._context = SSL.Context(SSL.SSLv23_METHOD) + self._context.set_verify(VERIFY_NONE, lambda *_: None) + + def getContext(self, hostname, port): + return self._context diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1c9e552788..b50a0c445c 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -57,14 +57,14 @@ incoming_responses_counter = metrics.register_counter( class MatrixFederationEndpointFactory(object): def __init__(self, hs): - self.tls_context_factory = hs.tls_context_factory + self.tls_server_context_factory = hs.tls_server_context_factory def endpointForURI(self, uri): destination = uri.netloc return matrix_federation_endpoint( reactor, destination, timeout=10, - ssl_context_factory=self.tls_context_factory + ssl_context_factory=self.tls_server_context_factory ) diff --git a/synapse/server.py b/synapse/server.py index 4d1fb1cbf6..656e534dff 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -19,7 +19,9 @@ # partial one for unit test mocking. # Imports required for the default HomeServer() implementation +from twisted.web.client import BrowserLikePolicyForHTTPS from synapse.federation import initialize_http_replication +from synapse.http.client import SimpleHttpClient, WoefullyInsecureContextFactory from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -87,6 +89,8 @@ class BaseHomeServer(object): 'pusherpool', 'event_builder_factory', 'filtering', + 'http_client_context_factory', + 'simple_http_client', ] def __init__(self, hostname, **kwargs): @@ -174,6 +178,16 @@ class HomeServer(BaseHomeServer): def build_auth(self): return Auth(self) + def build_http_client_context_factory(self): + config = self.get_config() + return ( + WoefullyInsecureContextFactory() if config.use_insecure_ssl_client + else BrowserLikePolicyForHTTPS() + ) + + def build_simple_http_client(self): + return SimpleHttpClient(self) + def build_v1auth(self): orf = Auth(self) # Matrix spec makes no reference to what HTTP status code is returned, -- cgit 1.5.1 From 6485f03d91a5f96da28f9dcc8e9ebc3adb213f6f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 13:05:00 +0100 Subject: Fix random formatting --- synapse/app/homeserver.py | 1 - synapse/http/client.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ba76ee362a..8e60304e29 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -15,7 +15,6 @@ # limitations under the License. import sys - sys.dont_write_bytecode = True from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS diff --git a/synapse/http/client.py b/synapse/http/client.py index da77c8b0ac..815a838729 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -265,7 +265,8 @@ class WoefullyInsecureContextFactory(ssl.ContextFactory): """ Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. - Do not use this unless you really, really hate your users.""" + Do not use this unless you really, really hate your users. + """ def __init__(self): self._context = SSL.Context(SSL.SSLv23_METHOD) -- cgit 1.5.1 From 89ae0166ded093be2343409cfe42f475dea83139 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 13:25:22 +0100 Subject: Allow room initialSync for users that have left the room, returning a snapshot of how the room was when they left it --- synapse/api/auth.py | 49 ++++++++++++++++++++++++++ synapse/handlers/message.py | 85 ++++++++++++++++++++++++++++++++++++++++----- synapse/storage/stream.py | 15 ++++++++ 3 files changed, 140 insertions(+), 9 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 0c0d678562..9b614a12bb 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -104,6 +104,20 @@ class Auth(object): @defer.inlineCallbacks def check_joined_room(self, room_id, user_id, current_state=None): + """Check if the user is currently joined in the room + Args: + room_id(str): The room to check. + user_id(str): The user to check. + current_state(dict): Optional map of the current state of the room. + If provided then that map is used to check whether they are a + member of the room. Otherwise the current membership is + loaded from the database. + Raises: + AuthError if the user is not in the room. + Returns: + A deferred membership event for the user if the user is in + the room. + """ if current_state: member = current_state.get( (EventTypes.Member, user_id), @@ -119,6 +133,41 @@ class Auth(object): self._check_joined_room(member, user_id, room_id) defer.returnValue(member) + @defer.inlineCallbacks + def check_user_was_in_room(self, room_id, user_id, current_state=None): + """Check if the user was in the room at some point. + Args: + room_id(str): The room to check. + user_id(str): The user to check. + current_state(dict): Optional map of the current state of the room. + If provided then that map is used to check whether they are a + member of the room. Otherwise the current membership is + loaded from the database. + Raises: + AuthError if the user was never in the room. + Returns: + A deferred membership event for the user if the user was in + the room. + """ + if current_state: + member = current_state.get( + (EventTypes.Member, user_id), + None + ) + else: + member = yield self.state.get_current_state( + room_id=room_id, + event_type=EventTypes.Member, + state_key=user_id + ) + + if not member: + raise AuthError(403, "User %s not in room %s" % ( + user_id, room_id + )) + + defer.returnValue(member) + @defer.inlineCallbacks def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5447c97e83..fc9a234333 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -22,7 +22,7 @@ from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator from synapse.util import unwrapFirstError from synapse.util.logcontext import PreserveLoggingContext -from synapse.types import UserID, RoomStreamToken +from synapse.types import UserID, RoomStreamToken, StreamToken from ._base import BaseHandler @@ -377,7 +377,6 @@ class MessageHandler(BaseHandler): lambda states: states[event.event_id] ) - (messages, token), current_state = yield defer.gatherResults( [ self.store.get_recent_events_for_room( @@ -434,13 +433,83 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def room_initial_sync(self, user_id, room_id, pagin_config=None, feedback=False): - current_state = yield self.state.get_current_state( - room_id=room_id, + """Capture the a snapshot of a room. If user is currently a member of + the room this will be what is currently in the room. If the user left + the room this will be what was in the room when they left. + + Args: + user_id(str): The user to get a snapshot for. + room_id(str): The room to get a snapshot of. + pagin_config(synapse.api.streams.PaginationConfig): The pagination + config used to determine how many messages to return. + Raises: + AuthError if the user wasn't in the room. + Returns: + A JSON object with the snapshot of the room. + """ + + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) + + if member_event.membership == Membership.JOIN: + result = yield self._room_initial_sync_joined( + user_id, room_id, pagin_config, member_event + ) + elif member_event.membership == Membership.LEAVE: + result = yield self._room_initial_sync_parted( + user_id, room_id, pagin_config, member_event + ) + defer.returnValue(result) + + @defer.inlineCallbacks + def _room_initial_sync_parted(self, user_id, room_id, pagin_config, + member_event): + room_state = yield self.store.get_state_for_events( + member_event.room_id, [member_event.event_id], None + ) + + room_state = room_state[member_event.event_id] + + limit = pagin_config.limit if pagin_config else None + if limit is None: + limit = 10 + + stream_token = yield self.store.get_stream_token_for_event( + member_event.event_id + ) + + messages, token = yield self.store.get_recent_events_for_room( + room_id, + limit=limit, + end_token=stream_token + ) + + messages = yield self._filter_events_for_client( + user_id, room_id, messages ) - yield self.auth.check_joined_room( - room_id, user_id, - current_state=current_state + start_token = StreamToken(token[0], 0, 0, 0) + end_token = StreamToken(token[1], 0, 0, 0) + + time_now = self.clock.time_msec() + + defer.returnValue({ + "membership": member_event.membership, + "room_id": room_id, + "messages": { + "chunk": [serialize_event(m, time_now) for m in messages], + "start": start_token.to_string(), + "end": end_token.to_string(), + }, + "state": [serialize_event(s, time_now) for s in room_state.values()], + "presence": [], + "receipts": [], + }) + + @defer.inlineCallbacks + def _room_initial_sync_joined(self, user_id, room_id, pagin_config, + member_event): + current_state = yield self.state.get_current_state( + room_id=room_id, ) # TODO(paul): I wish I was called with user objects not user_id @@ -454,8 +523,6 @@ class MessageHandler(BaseHandler): for x in current_state.values() ] - member_event = current_state.get((EventTypes.Member, user_id,)) - now_token = yield self.hs.get_event_sources().get_current_token() limit = pagin_config.limit if pagin_config else None diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index d7fe423f5a..0abfa86cd2 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -379,6 +379,21 @@ class StreamStore(SQLBaseStore): ) defer.returnValue("t%d-%d" % (topo, token)) + def get_stream_token_for_event(self, event_id): + """The stream token for an event + Args: + event_id(str): The id of the event to look up a stream token for. + Raises: + StoreError if the event wasn't in the database. + Returns: + A deferred "s%d" stream token. + """ + return self._simple_select_one_onecol( + table="events", + keyvalues={"event_id": event_id}, + retcol="stream_ordering", + ).addCallback(lambda stream_ordering: "s%d" % (stream_ordering,)) + def _get_max_topological_txn(self, txn): txn.execute( "SELECT MAX(topological_ordering) FROM events" -- cgit 1.5.1 From ddfe30ba835da4357670f2a2a39386b8b8e65b60 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 13:26:23 +0100 Subject: Better document the intent of the insecure SSL setting --- synapse/config/tls.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 472cf7ac4a..35ff13f4ba 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -42,9 +42,13 @@ class TlsConfig(Config): config.get("tls_dh_params_path"), "tls_dh_params" ) + # This config option applies to non-federation HTTP clients + # (e.g. for talking to recaptcha, identity servers, and such) + # It should never be used in production, and is intended for + # use only when running tests. self.use_insecure_ssl_client = config.get( - "i_really_want_to_ignore_ssl_certs_when_i_am_an_http_client_even_" - "though_it_is_woefully_insecure_because_i_hate_my_users", False) + "i_really_want_to_ignore_ssl_certs_when_i_am_an_https_client_even_" + "though_it_is_woefully_insecure_because_i_am_testing_i_promise", False) def default_config(self, config_dir_path, server_name): base_key_name = os.path.join(config_dir_path, server_name) -- cgit 1.5.1 From c0d1f37baf33aeeab22e635b5fd7905ab07e39e3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Sep 2015 13:47:14 +0100 Subject: Don't require pdus in check_auth script --- scripts-dev/check_auth.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts-dev/check_auth.py b/scripts-dev/check_auth.py index b889ac7fa7..4fa8792a5f 100644 --- a/scripts-dev/check_auth.py +++ b/scripts-dev/check_auth.py @@ -56,10 +56,9 @@ if __name__ == '__main__': js = json.load(args.json) - auth = Auth(Mock()) check_auth( auth, [FrozenEvent(d) for d in js["auth_chain"]], - [FrozenEvent(d) for d in js["pdus"]], + [FrozenEvent(d) for d in js.get("pdus", [])], ) -- cgit 1.5.1 From 1d579df66475c342e0bf5fed338808bdbfd03c94 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 14:12:24 +0100 Subject: Allow rooms/{roomId}/state for a room that has been left --- synapse/api/auth.py | 3 ++- synapse/handlers/message.py | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9b614a12bb..d8bb64a4af 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -160,8 +160,9 @@ class Auth(object): event_type=EventTypes.Member, state_key=user_id ) + membership = member.membership if member else None - if not member: + if membership not in (Membership.JOIN, Membership.LEAVE): raise AuthError(403, "User %s not in room %s" % ( user_id, room_id )) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index fc9a234333..171e9d72ac 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -277,7 +277,9 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def get_state_events(self, user_id, room_id): - """Retrieve all state events for a given room. + """Retrieve all state events for a given room. If the user is + joined to the room then return the current state. If the user has + left the room return the state events from when they left. Args: user_id(str): The user requesting state events. @@ -285,13 +287,19 @@ class MessageHandler(BaseHandler): Returns: A list of dicts representing state events. [{}, {}, {}] """ - yield self.auth.check_joined_room(room_id, user_id) + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) + + if member_event.membership == Membership.JOIN: + room_state = yield self.state_handler.get_current_state(room_id) + elif member_event.membership == Membership.LEAVE: + room_state = yield self.store.get_state_for_events( + room_id, [member_event.event_id], None + ) + room_state = room_state[member_event.event_id] - # TODO: This is duplicating logic from snapshot_all_rooms - current_state = yield self.state_handler.get_current_state(room_id) now = self.clock.time_msec() defer.returnValue( - [serialize_event(c, now) for c in current_state.values()] + [serialize_event(c, now) for c in room_state.values()] ) @defer.inlineCallbacks -- cgit 1.5.1 From bc8b25eb56bf4fcec3546c2ea28741189a519da5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 15:42:16 +0100 Subject: Allow users that have left the room to view the member list from the point they left --- synapse/handlers/room.py | 36 ------------------------------------ synapse/rest/client/v1/room.py | 18 +++++++++++++----- tests/rest/client/v1/test_rooms.py | 4 ++-- 3 files changed, 15 insertions(+), 43 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c5d1001b50..0ff816d53e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -25,7 +25,6 @@ from synapse.api.constants import ( from synapse.api.errors import StoreError, SynapseError from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor -from synapse.events.utils import serialize_event from collections import OrderedDict import logging @@ -342,41 +341,6 @@ class RoomMemberHandler(BaseHandler): if remotedomains is not None: remotedomains.add(member.domain) - @defer.inlineCallbacks - def get_room_members_as_pagination_chunk(self, room_id=None, user_id=None, - limit=0, start_tok=None, - end_tok=None): - """Retrieve a list of room members in the room. - - Args: - room_id (str): The room to get the member list for. - user_id (str): The ID of the user making the request. - limit (int): The max number of members to return. - start_tok (str): Optional. The start token if known. - end_tok (str): Optional. The end token if known. - Returns: - dict: A Pagination streamable dict. - Raises: - SynapseError if something goes wrong. - """ - yield self.auth.check_joined_room(room_id, user_id) - - member_list = yield self.store.get_room_members(room_id=room_id) - time_now = self.clock.time_msec() - event_list = [ - serialize_event(entry, time_now) - for entry in member_list - ] - chunk_data = { - "start": "START", # FIXME (erikj): START is no longer valid - "end": "END", - "chunk": event_list - } - # TODO honor Pagination stream params - # TODO snapshot this list to return on subsequent requests when - # paginating - defer.returnValue(chunk_data) - @defer.inlineCallbacks def change_membership(self, event, context, do_auth=True): """ Change the membership status of a user in a room. diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c9c27dd5a0..f4558b95a7 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -290,12 +290,18 @@ class RoomMemberListRestServlet(ClientV1RestServlet): def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) user, _ = yield self.auth.get_user_by_req(request) - handler = self.handlers.room_member_handler - members = yield handler.get_room_members_as_pagination_chunk( + handler = self.handlers.message_handler + events = yield handler.get_state_events( room_id=room_id, - user_id=user.to_string()) + user_id=user.to_string(), + ) + + chunk = [] - for event in members["chunk"]: + for event in events: + if event["type"] != EventTypes.Member: + continue + chunk.append(event) # FIXME: should probably be state_key here, not user_id target_user = UserID.from_string(event["user_id"]) # Presence is an optional cache; don't fail if we can't fetch it @@ -308,7 +314,9 @@ class RoomMemberListRestServlet(ClientV1RestServlet): except: pass - defer.returnValue((200, members)) + defer.returnValue((200, { + "chunk": chunk + })) # TODO: Needs unit testing diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 34ab47d02e..d50cfe4298 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -492,9 +492,9 @@ class RoomsMemberListTestCase(RestTestCase): self.assertEquals(200, code, msg=str(response)) yield self.leave(room=room_id, user=self.user_id) - # can no longer see list, you've left. + # can see old list once left (code, response) = yield self.mock_resource.trigger_get(room_path) - self.assertEquals(403, code, msg=str(response)) + self.assertEquals(200, code, msg=str(response)) class RoomsCreateTestCase(RestTestCase): -- cgit 1.5.1 From 3c166a24c591afdc851de3c6c754c90471b1b0a9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 16:05:09 +0100 Subject: Remove undocumented and unimplemented 'feedback' parameter from the Client-Server API --- synapse/api/constants.py | 11 ----------- synapse/handlers/message.py | 21 +++------------------ synapse/handlers/room.py | 1 - synapse/rest/client/v1/initial_sync.py | 2 -- synapse/rest/client/v1/room.py | 2 -- synapse/storage/stream.py | 10 ++-------- 6 files changed, 5 insertions(+), 42 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 1423986c1e..3385664394 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -27,16 +27,6 @@ class Membership(object): LIST = (INVITE, JOIN, KNOCK, LEAVE, BAN) -class Feedback(object): - - """Represents the types of feedback a user can send in response to a - message.""" - - DELIVERED = u"delivered" - READ = u"read" - LIST = (DELIVERED, READ) - - class PresenceState(object): """Represents the presence state of a user.""" OFFLINE = u"offline" @@ -73,7 +63,6 @@ class EventTypes(object): PowerLevels = "m.room.power_levels" Aliases = "m.room.aliases" Redaction = "m.room.redaction" - Feedback = "m.room.message.feedback" RoomHistoryVisibility = "m.room.history_visibility" CanonicalAlias = "m.room.canonical_alias" diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 171e9d72ac..72ebac047f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -71,7 +71,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def get_messages(self, user_id=None, room_id=None, pagin_config=None, - feedback=False, as_client_event=True): + as_client_event=True): """Get messages in a room. Args: @@ -79,7 +79,6 @@ class MessageHandler(BaseHandler): room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. - feedback (bool): True to get compressed feedback with the messages as_client_event (bool): True to get events in client-server format. Returns: dict: Pagination API results @@ -264,17 +263,6 @@ class MessageHandler(BaseHandler): ) defer.returnValue(data) - @defer.inlineCallbacks - def get_feedback(self, event_id): - # yield self.auth.check_joined_room(room_id, user_id) - - # Pull out the feedback from the db - fb = yield self.store.get_feedback(event_id) - - if fb: - defer.returnValue(fb) - defer.returnValue(None) - @defer.inlineCallbacks def get_state_events(self, user_id, room_id): """Retrieve all state events for a given room. If the user is @@ -303,8 +291,7 @@ class MessageHandler(BaseHandler): ) @defer.inlineCallbacks - def snapshot_all_rooms(self, user_id=None, pagin_config=None, - feedback=False, as_client_event=True): + def snapshot_all_rooms(self, user_id=None, pagin_config=None, as_client_event=True): """Retrieve a snapshot of all rooms the user is invited or has joined. This snapshot may include messages for all rooms where the user is @@ -314,7 +301,6 @@ class MessageHandler(BaseHandler): user_id (str): The ID of the user making the request. pagin_config (synapse.api.streams.PaginationConfig): The pagination config used to determine how many messages *PER ROOM* to return. - feedback (bool): True to get feedback along with these messages. as_client_event (bool): True to get events in client-server format. Returns: A list of dicts with "room_id" and "membership" keys for all rooms @@ -439,8 +425,7 @@ class MessageHandler(BaseHandler): defer.returnValue(ret) @defer.inlineCallbacks - def room_initial_sync(self, user_id, room_id, pagin_config=None, - feedback=False): + def room_initial_sync(self, user_id, room_id, pagin_config=None): """Capture the a snapshot of a room. If user is currently a member of the room this will be what is currently in the room. If the user left the room this will be what was in the room when they left. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0ff816d53e..243623190f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -610,7 +610,6 @@ class RoomEventSource(object): to_key=config.to_key, direction=config.direction, limit=config.limit, - with_feedback=True ) defer.returnValue((events, next_key)) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 4ea4da653c..bac68cc29f 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -26,14 +26,12 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): user, _ = yield self.auth.get_user_by_req(request) - with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) handler = self.handlers.message_handler content = yield handler.snapshot_all_rooms( user_id=user.to_string(), pagin_config=pagination_config, - feedback=with_feedback, as_client_event=as_client_event ) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f4558b95a7..23871f161e 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -329,14 +329,12 @@ class RoomMessageListRestServlet(ClientV1RestServlet): pagination_config = PaginationConfig.from_request( request, default_limit=10, ) - with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args handler = self.handlers.message_handler msgs = yield handler.get_messages( room_id=room_id, user_id=user.to_string(), pagin_config=pagination_config, - feedback=with_feedback, as_client_event=as_client_event ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 0abfa86cd2..5763c462af 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -159,9 +159,7 @@ class StreamStore(SQLBaseStore): @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, - limit=0, with_feedback=False): - # TODO (erikj): Handle compressed feedback - + limit=0): current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " " INNER JOIN current_state_events as c" @@ -227,10 +225,7 @@ class StreamStore(SQLBaseStore): @defer.inlineCallbacks def paginate_room_events(self, room_id, from_key, to_key=None, - direction='b', limit=-1, - with_feedback=False): - # TODO (erikj): Handle compressed feedback - + direction='b', limit=-1): # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. @@ -302,7 +297,6 @@ class StreamStore(SQLBaseStore): @cachedInlineCallbacks(num_args=4) def get_recent_events_for_room(self, room_id, limit, end_token, from_token=None): - # TODO (erikj): Handle compressed feedback end_token = RoomStreamToken.parse_stream_token(end_token) -- cgit 1.5.1 From dd0867f5ba3da5af191d82ccf1292842b21da84e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Sep 2015 17:02:39 +0100 Subject: Various bug fixes to crypto.keyring --- synapse/crypto/keyring.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index a692cdbe55..e251ab6af3 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -162,7 +162,9 @@ class Keyring(object): def remove_deferreds(res, server_name, group_id): server_to_gids[server_name].discard(group_id) if not server_to_gids[server_name]: - server_to_deferred.pop(server_name).callback(None) + d = server_to_deferred.pop(server_name, None) + if d: + d.callback(None) return res for g_id, deferred in deferreds.items(): @@ -200,8 +202,15 @@ class Keyring(object): else: break - for server_name, deferred in server_to_deferred: - self.key_downloads[server_name] = ObservableDeferred(deferred) + for server_name, deferred in server_to_deferred.items(): + d = ObservableDeferred(deferred) + self.key_downloads[server_name] = d + + def rm(r, server_name): + self.key_downloads.pop(server_name, None) + return r + + d.addBoth(rm, server_name) def get_server_verify_keys(self, group_id_to_group, group_id_to_deferred): """Takes a dict of KeyGroups and tries to find at least one key for @@ -220,9 +229,8 @@ class Keyring(object): merged_results = {} missing_keys = { - group.server_name: key_id + group.server_name: set(group.key_ids) for group in group_id_to_group.values() - for key_id in group.key_ids } for fn in key_fetch_fns: @@ -279,16 +287,15 @@ class Keyring(object): def get_keys_from_store(self, server_name_and_key_ids): res = yield defer.gatherResults( [ - self.store.get_server_verify_keys(server_name, key_ids) + self.store.get_server_verify_keys( + server_name, key_ids + ).addCallback(lambda ks, server: (server, ks), server_name) for server_name, key_ids in server_name_and_key_ids ], consumeErrors=True, ).addErrback(unwrapFirstError) - defer.returnValue(dict(zip( - [server_name for server_name, _ in server_name_and_key_ids], - res - ))) + defer.returnValue(dict(res)) @defer.inlineCallbacks def get_keys_from_perspectives(self, server_name_and_key_ids): -- cgit 1.5.1 From 09cb5c7d33c32e2cbf5a5b6f6f0e2780338491d2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 17:31:09 +0100 Subject: Allow users that have left a room to get the messages that happend in the room before they left --- synapse/handlers/message.py | 31 +++++++++++++++++++++++++++---- synapse/storage/stream.py | 19 ++++++++++++++++++- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 72ebac047f..db89491b46 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -83,21 +83,44 @@ class MessageHandler(BaseHandler): Returns: dict: Pagination API results """ - yield self.auth.check_joined_room(room_id, user_id) + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) data_source = self.hs.get_event_sources().sources["room"] - if not pagin_config.from_token: + if pagin_config.from_token: + room_token = pagin_config.from_token.room_key + else: pagin_config.from_token = ( yield self.hs.get_event_sources().get_current_token( direction='b' ) ) + room_token = pagin_config.from_token.room_key - room_token = RoomStreamToken.parse(pagin_config.from_token.room_key) + room_token = RoomStreamToken.parse(room_token) if room_token.topological is None: raise SynapseError(400, "Invalid token") + pagin_config.from_token = pagin_config.from_token.copy_and_replace( + "room_key", str(room_token) + ) + + source_config = pagin_config.get_source_config("room") + + if member_event.membership == Membership.LEAVE: + # If they have left the room then clamp the token to be before + # they left the room + leave_token = yield self.store.get_topological_token_for_event( + member_event.event_id + ) + leave_token = RoomStreamToken.parse(leave_token) + if leave_token.topological < room_token.topological: + source_config.from_key = str(leave_token) + + if source_config.direction == "f": + if source_config.to_key is None: + source_config.to_key = str(leave_token) + yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, room_token.topological ) @@ -105,7 +128,7 @@ class MessageHandler(BaseHandler): user = UserID.from_string(user_id) events, next_key = yield data_source.get_pagination_rows( - user, pagin_config.get_source_config("room"), room_id + user, source_config, room_id ) next_token = pagin_config.from_token.copy_and_replace( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 5763c462af..3cab06fdef 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -386,7 +386,24 @@ class StreamStore(SQLBaseStore): table="events", keyvalues={"event_id": event_id}, retcol="stream_ordering", - ).addCallback(lambda stream_ordering: "s%d" % (stream_ordering,)) + ).addCallback(lambda row: "s%d" % (row,)) + + def get_topological_token_for_event(self, event_id): + """The stream token for an event + Args: + event_id(str): The id of the event to look up a stream token for. + Raises: + StoreError if the event wasn't in the database. + Returns: + A deferred "t%d-%d" topological token. + """ + return self._simple_select_one( + table="events", + keyvalues={"event_id": event_id}, + retcols=("stream_ordering", "topological_ordering"), + ).addCallback(lambda row: "t%d-%d" % ( + row["topological_ordering"], row["stream_ordering"],) + ) def _get_max_topological_txn(self, txn): txn.execute( -- cgit 1.5.1 From 30768dcf4082d54101c0a6a9ac3d04462a0395ac Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 Sep 2015 10:33:48 +0100 Subject: Fix adding threepids to an existing account --- synapse/rest/client/v2_alpha/account.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 522a312c9e..6281e2d029 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -96,6 +96,7 @@ class ThreepidRestServlet(RestServlet): self.hs = hs self.identity_handler = hs.get_handlers().identity_handler self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler @defer.inlineCallbacks def on_GET(self, request): -- cgit 1.5.1 From 49ae42bbe1176b9061c17cf7e3829008f608c0a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 10 Sep 2015 14:25:54 +0100 Subject: Bundle in some room state in the unsigned bit of the invite when sending to invited servers --- synapse/events/utils.py | 5 ++++- synapse/handlers/_base.py | 29 +++++++++++++++++++++++++---- synapse/handlers/message.py | 4 ++++ synapse/storage/roommember.py | 4 ++-- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 7bd78343f0..b36eec0993 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -103,7 +103,10 @@ def format_event_raw(d): def format_event_for_client_v1(d): d["user_id"] = d.pop("sender", None) - move_keys = ("age", "redacted_because", "replaces_state", "prev_content") + move_keys = ( + "age", "redacted_because", "replaces_state", "prev_content", + "invite_room_state", + ) for key in move_keys: if key in d["unsigned"]: d[key] = d["unsigned"][key] diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 60ac6617ae..3a232cbeae 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -123,24 +123,38 @@ class BaseHandler(object): ) ) - (event_stream_id, max_stream_id) = yield self.store.persist_event( - event, context=context - ) - federation_handler = self.hs.get_handlers().federation_handler if event.type == EventTypes.Member: if event.content["membership"] == Membership.INVITE: + event.unsigned["invite_room_state"] = [ + { + "type": e.type, + "state_key": e.state_key, + "content": e.content, + } + for k, e in context.current_state.items() + if e.type in ( + EventTypes.JoinRules, + EventTypes.CanonicalAlias, + EventTypes.RoomAvatar, + EventTypes.Name, + ) + ] + invitee = UserID.from_string(event.state_key) if not self.hs.is_mine(invitee): # TODO: Can we add signature from remote server in a nicer # way? If we have been invited by a remote server, we need # to get them to sign the event. + returned_invite = yield federation_handler.send_invite( invitee.domain, event, ) + event.unsigned.pop("room_state", None) + # TODO: Make sure the signatures actually are correct. event.signatures.update( returned_invite.signatures @@ -161,6 +175,10 @@ class BaseHandler(object): "You don't have permission to redact events" ) + (event_stream_id, max_stream_id) = yield self.store.persist_event( + event, context=context + ) + destinations = set(extra_destinations) for k, s in context.current_state.items(): try: @@ -189,6 +207,9 @@ class BaseHandler(object): notify_d.addErrback(log_failure) + # If invite, remove room_state from unsigned before sending. + event.unsigned.pop("invite_room_state", None) + federation_handler.handle_new_event( event, destinations=destinations, ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 23b779ad7c..a5d9df8804 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -354,8 +354,12 @@ class MessageHandler(BaseHandler): } if event.membership == Membership.INVITE: + time_now = self.clock.time_msec() d["inviter"] = event.sender + invite_event = yield self.store.get_event(event.event_id) + d["invite"] = serialize_event(invite_event, time_now, as_client_event) + rooms_ret.append(d) if event.membership != Membership.JOIN: diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 8eee2dfbcc..2a59ee7d6d 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) RoomsForUser = namedtuple( "RoomsForUser", - ("room_id", "sender", "membership") + ("room_id", "sender", "membership", "event_id") ) @@ -141,7 +141,7 @@ class RoomMemberStore(SQLBaseStore): args.extend(membership_list) sql = ( - "SELECT m.room_id, m.sender, m.membership" + "SELECT m.event_id, m.room_id, m.sender, m.membership" " FROM room_memberships as m" " INNER JOIN current_state_events as c" " ON m.event_id = c.event_id " -- cgit 1.5.1 From e2054ce21a04f3d741293f50b283c01bbe2b0591 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 10 Sep 2015 15:06:47 +0100 Subject: Allow users to GET individual state events for rooms that they have left --- synapse/handlers/message.py | 20 +++++++++++++------- tests/rest/client/v1/test_rooms.py | 10 +++++----- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index db89491b46..5d18aaacf0 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import RoomError, SynapseError +from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator @@ -277,13 +277,19 @@ class MessageHandler(BaseHandler): Raises: SynapseError if something went wrong. """ - have_joined = yield self.auth.check_joined_room(room_id, user_id) - if not have_joined: - raise RoomError(403, "User not in room.") + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) + + if member_event.membership == Membership.JOIN: + data = yield self.state_handler.get_current_state( + room_id, event_type, state_key + ) + elif member_event.membership == Membership.LEAVE: + key = (event_type, state_key) + room_state = yield self.store.get_state_for_events( + room_id, [member_event.event_id], [key] + ) + data = room_state[member_event.event_id].get(key) - data = yield self.state_handler.get_current_state( - room_id, event_type, state_key - ) defer.returnValue(data) @defer.inlineCallbacks diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index d50cfe4298..ed0ac8d5c8 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -239,7 +239,7 @@ class RoomPermissionsTestCase(RestTestCase): "PUT", topic_path, topic_content) self.assertEquals(403, code, msg=str(response)) (code, response) = yield self.mock_resource.trigger_get(topic_path) - self.assertEquals(403, code, msg=str(response)) + self.assertEquals(200, code, msg=str(response)) # get topic in PUBLIC room, not joined, expect 403 (code, response) = yield self.mock_resource.trigger_get( @@ -301,11 +301,11 @@ class RoomPermissionsTestCase(RestTestCase): room=room, expect_code=200) # get membership of self, get membership of other, private room + left - # expect all 403s + # expect all 200s yield self.leave(room=room, user=self.user_id) yield self._test_get_membership( members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) + room=room, expect_code=200) @defer.inlineCallbacks def test_membership_public_room_perms(self): @@ -326,11 +326,11 @@ class RoomPermissionsTestCase(RestTestCase): room=room, expect_code=200) # get membership of self, get membership of other, public room + left - # expect 403. + # expect 200. yield self.leave(room=room, user=self.user_id) yield self._test_get_membership( members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) + room=room, expect_code=200) @defer.inlineCallbacks def test_invited_permissions(self): -- cgit 1.5.1 From dffc9c4ae0eea0616cc017c7f858f8a923202075 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 14 Sep 2015 14:41:37 +0100 Subject: Drop unused index --- synapse/storage/schema/delta/23/drop_state_index.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 synapse/storage/schema/delta/23/drop_state_index.sql diff --git a/synapse/storage/schema/delta/23/drop_state_index.sql b/synapse/storage/schema/delta/23/drop_state_index.sql new file mode 100644 index 0000000000..07d0ea5cb2 --- /dev/null +++ b/synapse/storage/schema/delta/23/drop_state_index.sql @@ -0,0 +1,16 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DROP INDEX IF EXISTS state_groups_state_tuple; -- cgit 1.5.1 From 9cd5b9a802f4c38f716a507a40188557103993f3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 14 Sep 2015 19:03:53 +0100 Subject: Hacky attempt at catching SIGHUP and rotating the logfile around --- synapse/config/logger.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index fa542623b7..daca698d0c 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -21,6 +21,7 @@ import logging.config import yaml from string import Template import os +import signal DEFAULT_LOG_CONFIG = Template(""" @@ -142,6 +143,19 @@ class LoggingConfig(Config): handler = logging.handlers.RotatingFileHandler( self.log_file, maxBytes=(1000 * 1000 * 100), backupCount=3 ) + + def sighup(signum, stack): + logger.info("Closing log file due to SIGHUP") + handler.doRollover() + logger.info("Opened new log file due to SIGHUP") + + # TODO(paul): obviously this is a terrible mechanism for + # stealing SIGHUP, because it means no other part of synapse + # can use it instead. If we want to catch SIGHUP anywhere + # else as well, I'd suggest we find a nicer way to broadcast + # it around. + if getattr(signal, "SIGHUP"): + signal.signal(signal.SIGHUP, sighup) else: handler = logging.StreamHandler() handler.setFormatter(formatter) -- cgit 1.5.1 From 3bcbabc9fb5446e74a675352e22963d528189957 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 15 Sep 2015 15:46:22 +0100 Subject: Rename context factory Mjark is officially no fun. --- synapse/http/client.py | 6 +++--- synapse/server.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 815a838729..0933388c04 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -261,11 +261,11 @@ def _print_ex(e): logger.exception(e) -class WoefullyInsecureContextFactory(ssl.ContextFactory): +class InsecureInterceptableContextFactory(ssl.ContextFactory): """ - Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. + Factory for PyOpenSSL SSL contexts which accepts any certificate for any domain. - Do not use this unless you really, really hate your users. + Do not use this since it allows an attacker to intercept your communications. """ def __init__(self): diff --git a/synapse/server.py b/synapse/server.py index 656e534dff..d96c5a573a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -21,7 +21,7 @@ # Imports required for the default HomeServer() implementation from twisted.web.client import BrowserLikePolicyForHTTPS from synapse.federation import initialize_http_replication -from synapse.http.client import SimpleHttpClient, WoefullyInsecureContextFactory +from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -181,7 +181,7 @@ class HomeServer(BaseHomeServer): def build_http_client_context_factory(self): config = self.get_config() return ( - WoefullyInsecureContextFactory() if config.use_insecure_ssl_client + InsecureInterceptableContextFactory() if config.use_insecure_ssl_client else BrowserLikePolicyForHTTPS() ) -- cgit 1.5.1 From d4af08a167cb5351110036c35bdfc267242d8131 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 15 Sep 2015 15:50:13 +0100 Subject: Use shorter config key name --- synapse/config/tls.py | 6 +++--- synapse/server.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 35ff13f4ba..e6023a718d 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -46,9 +46,9 @@ class TlsConfig(Config): # (e.g. for talking to recaptcha, identity servers, and such) # It should never be used in production, and is intended for # use only when running tests. - self.use_insecure_ssl_client = config.get( - "i_really_want_to_ignore_ssl_certs_when_i_am_an_https_client_even_" - "though_it_is_woefully_insecure_because_i_am_testing_i_promise", False) + self.use_insecure_ssl_client_just_for_testing_do_not_use = config.get( + "use_insecure_ssl_client_just_for_testing_do_not_use" + ) def default_config(self, config_dir_path, server_name): base_key_name = os.path.join(config_dir_path, server_name) diff --git a/synapse/server.py b/synapse/server.py index d96c5a573a..8424798b1b 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -181,7 +181,8 @@ class HomeServer(BaseHomeServer): def build_http_client_context_factory(self): config = self.get_config() return ( - InsecureInterceptableContextFactory() if config.use_insecure_ssl_client + InsecureInterceptableContextFactory() + if config.use_insecure_ssl_client_just_for_testing_do_not_use else BrowserLikePolicyForHTTPS() ) -- cgit 1.5.1 From ffe8cf7e59bab7c9da98534f27b1e486cb642223 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 11:51:25 +0100 Subject: Fix bug where we sometimes didn't fetch all the keys requested for a server. --- synapse/crypto/keyring.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 1b1b31c5c0..8b6a59866f 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -228,10 +228,9 @@ class Keyring(object): def do_iterations(): merged_results = {} - missing_keys = { - group.server_name: set(group.key_ids) - for group in group_id_to_group.values() - } + missing_keys = {} + for group in group_id_to_group.values(): + missing_keys.setdefault(group.server_name, set()).union(group.key_ids) for fn in key_fetch_fns: results = yield fn(missing_keys.items()) -- cgit 1.5.1 From 4678055173636f9940e77f1af35b888f99506030 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 11:07:22 +0100 Subject: Refactor do_invite_join --- synapse/handlers/federation.py | 84 ++++++++++++++++++++++++++++-------------- synapse/state.py | 3 -- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4ff20599d6..30b9982e25 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -649,35 +649,10 @@ class FederationHandler(BaseHandler): # FIXME pass - ev_infos = [] - for e in itertools.chain(state, auth_chain): - if e.event_id == event.event_id: - continue - - e.internal_metadata.outlier = True - auth_ids = [e_id for e_id, _ in e.auth_events] - ev_infos.append({ - "event": e, - "auth_events": { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - }) - - yield self._handle_new_events(origin, ev_infos, outliers=True) + self._check_auth_tree(auth_chain, event) - auth_ids = [e_id for e_id, _ in event.auth_events] - auth_events = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - - _, event_stream_id, max_stream_id = yield self._handle_new_event( - origin, - new_event, - state=state, - current_state=state, - auth_events=auth_events, + event_stream_id, max_stream_id = yield self._persist_auth_tree( + auth_chain, state, event ) with PreserveLoggingContext(): @@ -1026,6 +1001,59 @@ class FederationHandler(BaseHandler): is_new_state=(not outliers and not backfilled), ) + def _check_auth_tree(self, auth_events, event): + event_map = { + e.event_id: e + for e in auth_events + } + + create_event = None + for e in auth_events: + if (e.type, e.state_key) == (EventTypes.Create, ""): + create_event = e + break + + for e in auth_events + [event]: + a = { + (event_map[e_id].type, event_map[e_id].state_key): event_map[e_id] + for e_id, _ in e.auth_events + } + if create_event: + a[(EventTypes.Create, "")] = create_event + + self.auth.check(e, auth_events=a) + + @defer.inlineCallbacks + def _persist_auth_tree(self, auth_events, state, event): + events_to_context = {} + for e in auth_events: + ctx = yield self.state_handler.compute_event_context( + e, outlier=True, + ) + events_to_context[e.event_id] = ctx + e.internal_metadata.outlier = True + + yield self.store.persist_events( + [ + (e, events_to_context[e.event_id]) + for e in auth_events + ], + is_new_state=False, + ) + + new_event_context = yield self.state_handler.compute_event_context( + event, old_state=state, outlier=False, + ) + + event_stream_id, max_stream_id = yield self.store.persist_event( + event, new_event_context, + backfilled=False, + is_new_state=True, + current_state=state, + ) + + defer.returnValue((event_stream_id, max_stream_id)) + @defer.inlineCallbacks def _prep_event(self, origin, event, state=None, backfilled=False, current_state=None, auth_events=None): diff --git a/synapse/state.py b/synapse/state.py index 1fe4d066bd..ed36f844cb 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -17,7 +17,6 @@ from twisted.internet import defer from synapse.util.logutils import log_function -from synapse.util.async import run_on_reactor from synapse.util.caches.expiringcache import ExpiringCache from synapse.api.constants import EventTypes from synapse.api.errors import AuthError @@ -119,8 +118,6 @@ class StateHandler(object): Returns: an EventContext """ - yield run_on_reactor() - context = EventContext() if outlier: -- cgit 1.5.1 From a3e332af1930c103b8b2ece9d50edd94193761e4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 13:41:07 +0100 Subject: Don't bail out of joining if we encounter a rejected event --- scripts-dev/check_auth.py | 2 +- synapse/handlers/federation.py | 33 ++++++++++++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/scripts-dev/check_auth.py b/scripts-dev/check_auth.py index 4fa8792a5f..b362aad722 100644 --- a/scripts-dev/check_auth.py +++ b/scripts-dev/check_auth.py @@ -38,7 +38,7 @@ def check_auth(auth, auth_chain, events): print "Failed:", e.event_id, e.type, e.state_key print "Auth_events:", auth_events print ex - print json.dumps(e.get_dict(), sort_keys=True, indent=4) + # print json.dumps(e.get_dict(), sort_keys=True, indent=4) # raise print "Success:", e.event_id, e.type, e.state_key diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 30b9982e25..0f11fa390f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -649,8 +649,6 @@ class FederationHandler(BaseHandler): # FIXME pass - self._check_auth_tree(auth_chain, event) - event_stream_id, max_stream_id = yield self._persist_auth_tree( auth_chain, state, event ) @@ -1001,7 +999,16 @@ class FederationHandler(BaseHandler): is_new_state=(not outliers and not backfilled), ) - def _check_auth_tree(self, auth_events, event): + @defer.inlineCallbacks + def _persist_auth_tree(self, auth_events, state, event): + events_to_context = {} + for e in auth_events: + ctx = yield self.state_handler.compute_event_context( + e, outlier=True, + ) + events_to_context[e.event_id] = ctx + e.internal_metadata.outlier = True + event_map = { e.event_id: e for e in auth_events @@ -1021,17 +1028,17 @@ class FederationHandler(BaseHandler): if create_event: a[(EventTypes.Create, "")] = create_event - self.auth.check(e, auth_events=a) + try: + self.auth.check(e, auth_events=a) + except AuthError: + logger.warn( + "Rejecting %s because %s", + event.event_id, e.msg + ) - @defer.inlineCallbacks - def _persist_auth_tree(self, auth_events, state, event): - events_to_context = {} - for e in auth_events: - ctx = yield self.state_handler.compute_event_context( - e, outlier=True, - ) - events_to_context[e.event_id] = ctx - e.internal_metadata.outlier = True + if e == event: + raise + events_to_context[e.event_id].rejected = RejectedReason.AUTH_ERROR yield self.store.persist_events( [ -- cgit 1.5.1 From 744e7d2790380b20260c0740fc68f7f49d07136b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 14:26:15 +0100 Subject: Also handle state --- synapse/handlers/federation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0f11fa390f..b148af5390 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1002,7 +1002,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _persist_auth_tree(self, auth_events, state, event): events_to_context = {} - for e in auth_events: + for e in itertools.chain(auth_events, state): ctx = yield self.state_handler.compute_event_context( e, outlier=True, ) @@ -1020,7 +1020,7 @@ class FederationHandler(BaseHandler): create_event = e break - for e in auth_events + [event]: + for e in itertools.chain(auth_events, state, [event]): a = { (event_map[e_id].type, event_map[e_id].state_key): event_map[e_id] for e_id, _ in e.auth_events @@ -1033,7 +1033,7 @@ class FederationHandler(BaseHandler): except AuthError: logger.warn( "Rejecting %s because %s", - event.event_id, e.msg + e.event_id, e.msg ) if e == event: -- cgit 1.5.1 From 3a01901d6c59f540540e00835f0716dfa7f03846 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 14:28:57 +0100 Subject: Capture err --- synapse/handlers/federation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b148af5390..fd8a86ea99 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1030,10 +1030,10 @@ class FederationHandler(BaseHandler): try: self.auth.check(e, auth_events=a) - except AuthError: + except AuthError as err: logger.warn( "Rejecting %s because %s", - e.event_id, e.msg + e.event_id, err.msg ) if e == event: -- cgit 1.5.1 From 54e688277ad86b95345c3e8d1306c7e08b0ed484 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 14:32:31 +0100 Subject: Also persist state --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index fd8a86ea99..85fdf94d06 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1043,7 +1043,7 @@ class FederationHandler(BaseHandler): yield self.store.persist_events( [ (e, events_to_context[e.event_id]) - for e in auth_events + for e in itertools.chain(auth_events, state) ], is_new_state=False, ) -- cgit 1.5.1 From c34ffd2736a2042484a3593a0174df5e2d118252 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 16 Sep 2015 16:26:03 +0100 Subject: Fix getting an event for a room the server forgot it was in --- synapse/handlers/federation.py | 106 +++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 47 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 85fdf94d06..e79a82cfc0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -125,60 +125,72 @@ class FederationHandler(BaseHandler): ) if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") - current_state = state - event_ids = set() - if state: - event_ids |= {e.event_id for e in state} - if auth_chain: - event_ids |= {e.event_id for e in auth_chain} + try: + event_stream_id, max_stream_id = yield self._persist_auth_tree( + auth_chain, state, event + ) + except AuthError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) - seen_ids = set( - (yield self.store.have_events(event_ids)).keys() - ) + else: + event_ids = set() + if state: + event_ids |= {e.event_id for e in state} + if auth_chain: + event_ids |= {e.event_id for e in auth_chain} + + seen_ids = set( + (yield self.store.have_events(event_ids)).keys() + ) - if state and auth_chain is not None: - # If we have any state or auth_chain given to us by the replication - # layer, then we should handle them (if we haven't before.) + if state and auth_chain is not None: + # If we have any state or auth_chain given to us by the replication + # layer, then we should handle them (if we haven't before.) - event_infos = [] + event_infos = [] - for e in itertools.chain(auth_chain, state): - if e.event_id in seen_ids: - continue - e.internal_metadata.outlier = True - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - event_infos.append({ - "event": e, - "auth_events": auth, - }) - seen_ids.add(e.event_id) + for e in itertools.chain(auth_chain, state): + if e.event_id in seen_ids: + continue + e.internal_metadata.outlier = True + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + event_infos.append({ + "event": e, + "auth_events": auth, + }) + seen_ids.add(e.event_id) - yield self._handle_new_events( - origin, - event_infos, - outliers=True - ) + yield self._handle_new_events( + origin, + event_infos, + outliers=True + ) - try: - _, event_stream_id, max_stream_id = yield self._handle_new_event( - origin, - event, - state=state, - backfilled=backfilled, - current_state=current_state, - ) - except AuthError as e: - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event.event_id, - ) + try: + _, event_stream_id, max_stream_id = yield self._handle_new_event( + origin, + event, + state=state, + backfilled=backfilled, + current_state=current_state, + ) + except AuthError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) # if we're receiving valid events from an origin, # it's probably a good idea to mark it as not in retry-state -- cgit 1.5.1 From 51b2448e050d4944d1a5176bcfbf30a33953ca68 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 17 Sep 2015 10:11:15 +0100 Subject: Revert change of scripts/check_auth.py --- scripts-dev/check_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-dev/check_auth.py b/scripts-dev/check_auth.py index b362aad722..4fa8792a5f 100644 --- a/scripts-dev/check_auth.py +++ b/scripts-dev/check_auth.py @@ -38,7 +38,7 @@ def check_auth(auth, auth_chain, events): print "Failed:", e.event_id, e.type, e.state_key print "Auth_events:", auth_events print ex - # print json.dumps(e.get_dict(), sort_keys=True, indent=4) + print json.dumps(e.get_dict(), sort_keys=True, indent=4) # raise print "Success:", e.event_id, e.type, e.state_key -- cgit 1.5.1 From b105996fc16ac264457c5a9f8e00c6a8263142d6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 17 Sep 2015 10:28:36 +0100 Subject: Remove run_on_reactor --- synapse/state.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 1fe4d066bd..ed36f844cb 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -17,7 +17,6 @@ from twisted.internet import defer from synapse.util.logutils import log_function -from synapse.util.async import run_on_reactor from synapse.util.caches.expiringcache import ExpiringCache from synapse.api.constants import EventTypes from synapse.api.errors import AuthError @@ -119,8 +118,6 @@ class StateHandler(object): Returns: an EventContext """ - yield run_on_reactor() - context = EventContext() if outlier: -- cgit 1.5.1 From 8e3bbc9bd01b8d0bd75e28d9c8862e555384e124 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 13:47:44 +0100 Subject: Clarify which event is returned by check_user_was_in_room --- synapse/api/auth.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d8bb64a4af..49a068afb1 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -146,8 +146,9 @@ class Auth(object): Raises: AuthError if the user was never in the room. Returns: - A deferred membership event for the user if the user was in - the room. + A deferred membership event for the user if the user was in the + room. This will be the join event if they are currently joined to + the room. This will be the leave event if they have left the room. """ if current_state: member = current_state.get( -- cgit 1.5.1 From 1e101ed4a45bad68f4865229e0c8d617fed67065 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:13:10 +0100 Subject: Clamp the "to" token for /rooms/{roomId}/messages to when the user left the room. There isn't a way for the client to learn a valid "to" token for a room that they have left in the C-S API but that doesn't stop a client making one up. --- synapse/handlers/message.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5d18aaacf0..bca592f5d7 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -120,6 +120,10 @@ class MessageHandler(BaseHandler): if source_config.direction == "f": if source_config.to_key is None: source_config.to_key = str(leave_token) + else: + to_token = RoomStreamToken.parse(source_config.to_key) + if leave_token.topological < to_token.topological: + source_config.to_key = str(leave_token) yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, room_token.topological -- cgit 1.5.1 From 0c162859897cb0cae5e501109e3d51d0e861e194 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:17:16 +0100 Subject: Add explicit "elif event.membership == Membership.LEAVE" for clarity --- synapse/handlers/message.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bca592f5d7..5593a8d4f7 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -389,13 +389,14 @@ class MessageHandler(BaseHandler): if event.membership not in (Membership.JOIN, Membership.LEAVE): return + try: if event.membership == Membership.JOIN: room_end_token = now_token.room_key deferred_room_state = self.state_handler.get_current_state( event.room_id ) - else: + elif event.membership == Membership.LEAVE: room_end_token = "s%d" % (event.stream_ordering,) deferred_room_state = self.store.get_state_for_events( event.room_id, [event.event_id], None -- cgit 1.5.1 From 95c304e3f93bc1113c2b4ac64d85c9fdeb7120b9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:18:47 +0100 Subject: Fix doc string to point at the right class --- synapse/handlers/message.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5593a8d4f7..bb73286729 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -467,8 +467,9 @@ class MessageHandler(BaseHandler): Args: user_id(str): The user to get a snapshot for. room_id(str): The room to get a snapshot of. - pagin_config(synapse.api.streams.PaginationConfig): The pagination - config used to determine how many messages to return. + pagin_config(synapse.streams.config.PaginationConfig): + The pagination config used to determine how many messages to + return. Raises: AuthError if the user wasn't in the room. Returns: -- cgit 1.5.1 From 49c0a0b5c4c2385fdf6755b1a5e1a3f0b04ef503 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:21:03 +0100 Subject: Clarify that room_initial_sync returns a python dict --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bb73286729..bda8eb5f3f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -473,7 +473,7 @@ class MessageHandler(BaseHandler): Raises: AuthError if the user wasn't in the room. Returns: - A JSON object with the snapshot of the room. + A JSON serialisable dict with the snapshot of the room. """ member_event = yield self.auth.check_user_was_in_room(room_id, user_id) -- cgit 1.5.1 From 7213588083dd9a721b0cd623fe22b308f25f19a5 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 12:57:40 +0100 Subject: Implement configurable stats reporting SYN-287 This requires that HS owners either opt in or out of stats reporting. When --generate-config is passed, --report-stats must be specified If an already-generated config is used, and doesn't have the report_stats key, it is requested to be set. --- synapse/app/homeserver.py | 35 ++++++- synapse/app/synctl.py | 12 ++- synapse/config/_base.py | 45 +++++++- synapse/config/appservice.py | 2 +- synapse/config/captcha.py | 2 +- synapse/config/database.py | 2 +- synapse/config/key.py | 2 +- synapse/config/logger.py | 2 +- synapse/config/metrics.py | 8 +- synapse/config/ratelimiting.py | 2 +- synapse/config/registration.py | 2 +- synapse/config/repository.py | 2 +- synapse/config/saml2.py | 2 +- synapse/config/server.py | 2 +- synapse/config/tls.py | 2 +- synapse/config/voip.py | 2 +- synapse/storage/__init__.py | 20 +++- synapse/storage/events.py | 58 ++++++++++- synapse/storage/registration.py | 12 +++ .../storage/schema/delta/24/stats_reporting.sql | 22 ++++ tests/storage/event_injector.py | 81 ++++++++++++++ tests/storage/test_events.py | 116 +++++++++++++++++++++ tests/storage/test_room.py | 2 +- tests/storage/test_stream.py | 68 +++--------- 24 files changed, 425 insertions(+), 78 deletions(-) create mode 100644 synapse/storage/schema/delta/24/stats_reporting.sql create mode 100644 tests/storage/event_injector.py create mode 100644 tests/storage/test_events.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 15c0a4a003..b4429bd4f3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -42,7 +42,7 @@ from synapse.storage import ( from synapse.server import HomeServer -from twisted.internet import reactor +from twisted.internet import reactor, task, defer from twisted.application import service from twisted.enterprise import adbapi from twisted.web.resource import Resource, EncodingResourceWrapper @@ -677,6 +677,39 @@ def run(hs): ThreadPool._worker = profile(ThreadPool._worker) reactor.run = profile(reactor.run) + start_time = hs.get_clock().time() + + @defer.inlineCallbacks + def phone_stats_home(): + now = int(hs.get_clock().time()) + uptime = int(now - start_time) + if uptime < 0: + uptime = 0 + + stats = {} + stats["homeserver"] = hs.config.server_name + stats["timestamp"] = now + stats["uptime_seconds"] = uptime + stats["total_users"] = yield hs.get_datastore().count_all_users() + + all_rooms = yield hs.get_datastore().get_rooms(False) + stats["total_room_count"] = len(all_rooms) + + stats["daily_active_users"] = yield hs.get_datastore().count_daily_users() + daily_messages = yield hs.get_datastore().count_daily_messages() + if daily_messages is not None: + stats["daily_messages"] = daily_messages + + logger.info("Reporting stats to matrix.org: %s" % (stats,)) + hs.get_simple_http_client().put_json( + "https://matrix.org/report-usage-stats/push", + stats + ) + + if hs.config.report_stats: + phone_home_task = task.LoopingCall(phone_stats_home) + phone_home_task.start(60 * 60 * 24, now=False) + def in_thread(): with LoggingContext("run"): change_resource_limit(hs.config.soft_file_limit) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 1f7d543c31..6bcc437591 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -25,6 +25,7 @@ SYNAPSE = ["python", "-B", "-m", "synapse.app.homeserver"] CONFIGFILE = "homeserver.yaml" GREEN = "\x1b[1;32m" +RED = "\x1b[1;31m" NORMAL = "\x1b[m" if not os.path.exists(CONFIGFILE): @@ -45,8 +46,15 @@ def start(): print "Starting ...", args = SYNAPSE args.extend(["--daemonize", "-c", CONFIGFILE]) - subprocess.check_call(args) - print GREEN + "started" + NORMAL + try: + subprocess.check_call(args) + print GREEN + "started" + NORMAL + except subprocess.CalledProcessError as e: + print ( + RED + + "error starting (exit code: %d); see above for logs" % e.returncode + + NORMAL + ) def stop(): diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 8a75c48733..b9983f72a2 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -26,6 +26,16 @@ class ConfigError(Exception): class Config(object): + stats_reporting_begging_spiel = ( + "We would really appreciate it if you could help our project out by " + "reporting anonymized usage statistics from your homeserver. Only very " + "basic aggregate data (e.g. number of users) will be reported, but it " + "helps us to track the growth of the Matrix community, and helps us to " + "make Matrix a success, as well as to convince other networks that they " + "should peer with us.\n" + "Thank you." + ) + @staticmethod def parse_size(value): if isinstance(value, int) or isinstance(value, long): @@ -111,11 +121,14 @@ class Config(object): results.append(getattr(cls, name)(self, *args, **kargs)) return results - def generate_config(self, config_dir_path, server_name): + def generate_config(self, config_dir_path, server_name, report_stats=None): default_config = "# vim:ft=yaml\n" default_config += "\n\n".join(dedent(conf) for conf in self.invoke_all( - "default_config", config_dir_path, server_name + "default_config", + config_dir_path=config_dir_path, + server_name=server_name, + report_stats=report_stats, )) config = yaml.load(default_config) @@ -139,6 +152,12 @@ class Config(object): action="store_true", help="Generate a config file for the server name" ) + config_parser.add_argument( + "--report-stats", + action="store", + help="Stuff", + choices=["yes", "no"] + ) config_parser.add_argument( "--generate-keys", action="store_true", @@ -189,6 +208,11 @@ class Config(object): config_files.append(config_path) if config_args.generate_config: + if config_args.report_stats is None: + config_parser.error( + "Please specify either --report-stats=yes or --report-stats=no\n\n" + + cls.stats_reporting_begging_spiel + ) if not config_files: config_parser.error( "Must supply a config file.\nA config file can be automatically" @@ -211,7 +235,9 @@ class Config(object): os.makedirs(config_dir_path) with open(config_path, "wb") as config_file: config_bytes, config = obj.generate_config( - config_dir_path, server_name + config_dir_path=config_dir_path, + server_name=server_name, + report_stats=(config_args.report_stats == "yes"), ) obj.invoke_all("generate_files", config) config_file.write(config_bytes) @@ -261,9 +287,20 @@ class Config(object): specified_config.update(yaml_config) server_name = specified_config["server_name"] - _, config = obj.generate_config(config_dir_path, server_name) + _, config = obj.generate_config( + config_dir_path=config_dir_path, + server_name=server_name + ) config.pop("log_config") config.update(specified_config) + if "report_stats" not in config: + sys.stderr.write( + "Please opt in or out of reporting anonymized homeserver usage " + "statistics, by setting the report_stats key in your config file " + " ( " + config_path + " ) " + + "to either True or False.\n\n" + + Config.stats_reporting_begging_spiel + "\n") + sys.exit(1) if generate_keys: obj.invoke_all("generate_files", config) diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 38f41933b7..b8d301995e 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -20,7 +20,7 @@ class AppServiceConfig(Config): def read_config(self, config): self.app_service_config_files = config.get("app_service_config_files", []) - def default_config(cls, config_dir_path, server_name): + def default_config(cls, **kwargs): return """\ # A list of application service config file to use app_service_config_files: [] diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 15a132b4e3..dd92fcd0dc 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -24,7 +24,7 @@ class CaptchaConfig(Config): self.captcha_bypass_secret = config.get("captcha_bypass_secret") self.recaptcha_siteverify_api = config["recaptcha_siteverify_api"] - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): return """\ ## Captcha ## diff --git a/synapse/config/database.py b/synapse/config/database.py index f0611e8884..baeda8f300 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -45,7 +45,7 @@ class DatabaseConfig(Config): self.set_databasepath(config.get("database_path")) - def default_config(self, config, config_dir_path): + def default_config(self, **kwargs): database_path = self.abspath("homeserver.db") return """\ # Database configuration diff --git a/synapse/config/key.py b/synapse/config/key.py index 23ac8a3fca..2c187065e5 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -40,7 +40,7 @@ class KeyConfig(Config): config["perspectives"] ) - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) return """\ ## Signing Keys ## diff --git a/synapse/config/logger.py b/synapse/config/logger.py index daca698d0c..bd0c17c861 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -70,7 +70,7 @@ class LoggingConfig(Config): self.log_config = self.abspath(config.get("log_config")) self.log_file = self.abspath(config.get("log_file")) - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): log_file = self.abspath("homeserver.log") log_config = self.abspath( os.path.join(config_dir_path, server_name + ".log.config") diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index ae5a691527..825fec9a38 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -19,13 +19,15 @@ from ._base import Config class MetricsConfig(Config): def read_config(self, config): self.enable_metrics = config["enable_metrics"] + self.report_stats = config.get("report_stats", None) self.metrics_port = config.get("metrics_port") self.metrics_bind_host = config.get("metrics_bind_host", "127.0.0.1") - def default_config(self, config_dir_path, server_name): - return """\ + def default_config(self, report_stats=None, **kwargs): + suffix = "" if report_stats is None else "report_stats: %(report_stats)s\n" + return ("""\ ## Metrics ### # Enable collection and rendering of performance metrics enable_metrics: False - """ + """ + suffix) % locals() diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 76d9970e5b..611b598ec7 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -27,7 +27,7 @@ class RatelimitConfig(Config): self.federation_rc_reject_limit = config["federation_rc_reject_limit"] self.federation_rc_concurrent = config["federation_rc_concurrent"] - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): return """\ ## Ratelimiting ## diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 62de4b399f..fa98eced34 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -34,7 +34,7 @@ class RegistrationConfig(Config): self.registration_shared_secret = config.get("registration_shared_secret") self.macaroon_secret_key = config.get("macaroon_secret_key") - def default_config(self, config_dir, server_name): + def default_config(self, **kwargs): registration_shared_secret = random_string_with_symbols(50) macaroon_secret_key = random_string_with_symbols(50) return """\ diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 64644b9a7a..2fcf872449 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -60,7 +60,7 @@ class ContentRepositoryConfig(Config): config["thumbnail_sizes"] ) - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): media_store = self.default_path("media_store") uploads_path = self.default_path("uploads") return """ diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py index 1532036876..4c6133cf22 100644 --- a/synapse/config/saml2.py +++ b/synapse/config/saml2.py @@ -41,7 +41,7 @@ class SAML2Config(Config): self.saml2_config_path = None self.saml2_idp_redirect_url = None - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): return """ # Enable SAML2 for registration and login. Uses pysaml2 # config_path: Path to the sp_conf.py configuration file diff --git a/synapse/config/server.py b/synapse/config/server.py index a03e55c223..4d12d49857 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -117,7 +117,7 @@ class ServerConfig(Config): self.content_addr = content_addr - def default_config(self, config_dir_path, server_name): + def default_config(self, server_name, **kwargs): if ":" in server_name: bind_port = int(server_name.split(":")[1]) unsecure_port = bind_port - 400 diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e6023a718d..0ac2698293 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -50,7 +50,7 @@ class TlsConfig(Config): "use_insecure_ssl_client_just_for_testing_do_not_use" ) - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) tls_certificate_path = base_key_name + ".tls.crt" diff --git a/synapse/config/voip.py b/synapse/config/voip.py index a1707223d3..a093354ccd 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -22,7 +22,7 @@ class VoipConfig(Config): self.turn_shared_secret = config["turn_shared_secret"] self.turn_user_lifetime = self.parse_duration(config["turn_user_lifetime"]) - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): return """\ ## Turn ## diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 77cb1dbd81..b64c90d631 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 23 +SCHEMA_VERSION = 24 dir_path = os.path.abspath(os.path.dirname(__file__)) @@ -126,6 +126,24 @@ class DataStore(RoomMemberStore, RoomStore, lock=False, ) + @defer.inlineCallbacks + def count_daily_users(self): + def _count_users(txn): + txn.execute( + "SELECT COUNT(DISTINCT user_id) AS users" + " FROM user_ips" + " WHERE last_seen > ?", + # This is close enough to a day for our purposes. + (int(self._clock.time_msec()) - (1000 * 60 * 60 * 24),) + ) + rows = self.cursor_to_dict(txn) + if rows: + return rows[0]["users"] + return 0 + + ret = yield self.runInteraction("count_users", _count_users) + defer.returnValue(ret) + def get_user_ip_and_agents(self, user): return self._simple_select_list( table="user_ips", diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0a477e3122..2b51db9940 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from _base import SQLBaseStore, _RollbackButIsFineException from twisted.internet import defer, reactor @@ -28,6 +27,7 @@ from canonicaljson import encode_canonical_json from contextlib import contextmanager import logging +import math import ujson as json logger = logging.getLogger(__name__) @@ -905,3 +905,59 @@ class EventsStore(SQLBaseStore): txn.execute(sql, (event.event_id,)) result = txn.fetchone() return result[0] if result else None + + @defer.inlineCallbacks + def count_daily_messages(self): + def _count_messages(txn): + now = self.hs.get_clock().time() + + txn.execute( + "SELECT reported_stream_token, reported_time FROM stats_reporting" + ) + last_reported = self.cursor_to_dict(txn) + + txn.execute( + "SELECT stream_ordering" + " FROM events" + " ORDER BY stream_ordering DESC" + " LIMIT 1" + ) + now_reporting = self.cursor_to_dict(txn) + if not now_reporting: + return None + now_reporting = now_reporting[0]["stream_ordering"] + + txn.execute("DELETE FROM stats_reporting") + txn.execute( + "INSERT INTO stats_reporting" + " (reported_stream_token, reported_time)" + " VALUES (?, ?)", + (now_reporting, now,) + ) + + if not last_reported: + return None + + # Close enough to correct for our purposes. + yesterday = (now - 24 * 60 * 60) + if math.fabs(yesterday - last_reported[0]["reported_time"]) > 60 * 60: + return None + + txn.execute( + "SELECT COUNT(*) as messages" + " FROM events NATURAL JOIN event_json" + " WHERE json like '%m.room.message%'" + " AND stream_ordering > ?" + " AND stream_ordering <= ?", + ( + last_reported[0]["reported_stream_token"], + now_reporting, + ) + ) + rows = self.cursor_to_dict(txn) + if not rows: + return None + return rows[0]["messages"] + + ret = yield self.runInteraction("count_messages", _count_messages) + defer.returnValue(ret) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index c9ceb132ae..6d76237658 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -289,3 +289,15 @@ class RegistrationStore(SQLBaseStore): if ret: defer.returnValue(ret['user_id']) defer.returnValue(None) + + @defer.inlineCallbacks + def count_all_users(self): + def _count_users(txn): + txn.execute("SELECT COUNT(*) AS users FROM users") + rows = self.cursor_to_dict(txn) + if rows: + return rows[0]["users"] + return 0 + + ret = yield self.runInteraction("count_users", _count_users) + defer.returnValue(ret) diff --git a/synapse/storage/schema/delta/24/stats_reporting.sql b/synapse/storage/schema/delta/24/stats_reporting.sql new file mode 100644 index 0000000000..e9165d2917 --- /dev/null +++ b/synapse/storage/schema/delta/24/stats_reporting.sql @@ -0,0 +1,22 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Should only ever contain one row +CREATE TABLE IF NOT EXISTS stats_reporting( + -- The stream ordering token which was most recently reported as stats + reported_stream_token INTEGER, + -- The time (seconds since epoch) stats were most recently reported + reported_time BIGINT +); diff --git a/tests/storage/event_injector.py b/tests/storage/event_injector.py new file mode 100644 index 0000000000..42bd8928bd --- /dev/null +++ b/tests/storage/event_injector.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest +from twisted.internet import defer + +from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID, RoomID + +from tests.utils import setup_test_homeserver + +from mock import Mock + + +class EventInjector: + def __init__(self, hs): + self.hs = hs + self.store = hs.get_datastore() + self.message_handler = hs.get_handlers().message_handler + self.event_builder_factory = hs.get_event_builder_factory() + + @defer.inlineCallbacks + def create_room(self, room): + builder = self.event_builder_factory.new({ + "type": EventTypes.Create, + "room_id": room.to_string(), + "content": {}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder + ) + + yield self.store.persist_event(event, context) + + @defer.inlineCallbacks + def inject_room_member(self, room, user, membership): + builder = self.event_builder_factory.new({ + "type": EventTypes.Member, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"membership": membership}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder + ) + + yield self.store.persist_event(event, context) + + defer.returnValue(event) + + @defer.inlineCallbacks + def inject_message(self, room, user, body): + builder = self.event_builder_factory.new({ + "type": EventTypes.Message, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"body": body, "msgtype": u"message"}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder + ) + + yield self.store.persist_event(event, context) diff --git a/tests/storage/test_events.py b/tests/storage/test_events.py new file mode 100644 index 0000000000..313013009e --- /dev/null +++ b/tests/storage/test_events.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import uuid +from mock.mock import Mock +from synapse.types import RoomID, UserID + +from tests import unittest +from twisted.internet import defer +from tests.storage.event_injector import EventInjector + +from tests.utils import setup_test_homeserver + + +class EventsStoreTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + self.hs = yield setup_test_homeserver( + resource_for_federation=Mock(), + http_client=None, + ) + self.store = self.hs.get_datastore() + self.db_pool = self.hs.get_db_pool() + self.message_handler = self.hs.get_handlers().message_handler + self.event_injector = EventInjector(self.hs) + + @defer.inlineCallbacks + def test_count_daily_messages(self): + self.db_pool.runQuery("DELETE FROM stats_reporting") + + self.hs.clock.now = 100 + + # Never reported before, and nothing which could be reported + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + count = yield self.db_pool.runQuery("SELECT COUNT(*) FROM stats_reporting") + self.assertEqual([(0,)], count) + + # Create something to report + room = RoomID.from_string("!abc123:test") + user = UserID.from_string("@raccoonlover:test") + yield self.event_injector.create_room(room) + + self.base_event = yield self._get_last_stream_token() + + yield self.event_injector.inject_message(room, user, "Raccoons are really cute") + + # Never reported before, something could be reported, but isn't because + # it isn't old enough. + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + self._assert_stats_reporting(1, self.hs.clock.now) + + # Already reported yesterday, two new events from today. + yield self.event_injector.inject_message(room, user, "Yeah they are!") + yield self.event_injector.inject_message(room, user, "Incredibly!") + self.hs.clock.now += 60 * 60 * 24 + count = yield self.store.count_daily_messages() + self.assertEqual(2, count) # 2 since yesterday + self._assert_stats_reporting(3, self.hs.clock.now) # 3 ever + + # Last reported too recently. + yield self.event_injector.inject_message(room, user, "Who could disagree?") + self.hs.clock.now += 60 * 60 * 22 + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + self._assert_stats_reporting(4, self.hs.clock.now) + + # Last reported too long ago + yield self.event_injector.inject_message(room, user, "No one.") + self.hs.clock.now += 60 * 60 * 26 + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + self._assert_stats_reporting(5, self.hs.clock.now) + + # And now let's actually report something + yield self.event_injector.inject_message(room, user, "Indeed.") + yield self.event_injector.inject_message(room, user, "Indeed.") + yield self.event_injector.inject_message(room, user, "Indeed.") + # A little over 24 hours is fine :) + self.hs.clock.now += (60 * 60 * 24) + 50 + count = yield self.store.count_daily_messages() + self.assertEqual(3, count) + self._assert_stats_reporting(8, self.hs.clock.now) + + @defer.inlineCallbacks + def _get_last_stream_token(self): + rows = yield self.db_pool.runQuery( + "SELECT stream_ordering" + " FROM events" + " ORDER BY stream_ordering DESC" + " LIMIT 1" + ) + if not rows: + defer.returnValue(0) + else: + defer.returnValue(rows[0][0]) + + @defer.inlineCallbacks + def _assert_stats_reporting(self, messages, time): + rows = yield self.db_pool.runQuery( + "SELECT reported_stream_token, reported_time FROM stats_reporting" + ) + self.assertEqual([(self.base_event + messages, time,)], rows) diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index ab7625a3ca..caffce64e3 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -85,7 +85,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): # Room events need the full datastore, for persist_event() and # get_room_state() self.store = hs.get_datastore() - self.event_factory = hs.get_event_factory(); + self.event_factory = hs.get_event_factory() self.room = RoomID.from_string("!abcde:test") diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 0c9b89d765..a658a789aa 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID +from tests.storage.event_injector import EventInjector from tests.utils import setup_test_homeserver @@ -36,6 +37,7 @@ class StreamStoreTestCase(unittest.TestCase): self.store = hs.get_datastore() self.event_builder_factory = hs.get_event_builder_factory() + self.event_injector = EventInjector(hs) self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler @@ -45,60 +47,20 @@ class StreamStoreTestCase(unittest.TestCase): self.room1 = RoomID.from_string("!abc123:test") self.room2 = RoomID.from_string("!xyx987:test") - self.depth = 1 - - @defer.inlineCallbacks - def inject_room_member(self, room, user, membership): - self.depth += 1 - - builder = self.event_builder_factory.new({ - "type": EventTypes.Member, - "sender": user.to_string(), - "state_key": user.to_string(), - "room_id": room.to_string(), - "content": {"membership": membership}, - }) - - event, context = yield self.message_handler._create_new_client_event( - builder - ) - - yield self.store.persist_event(event, context) - - defer.returnValue(event) - - @defer.inlineCallbacks - def inject_message(self, room, user, body): - self.depth += 1 - - builder = self.event_builder_factory.new({ - "type": EventTypes.Message, - "sender": user.to_string(), - "state_key": user.to_string(), - "room_id": room.to_string(), - "content": {"body": body, "msgtype": u"message"}, - }) - - event, context = yield self.message_handler._create_new_client_event( - builder - ) - - yield self.store.persist_event(event, context) - @defer.inlineCallbacks def test_event_stream_get_other(self): # Both bob and alice joins the room - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) # Initial stream key: start = yield self.store.get_room_events_max_id() - yield self.inject_message(self.room1, self.u_alice, u"test") + yield self.event_injector.inject_message(self.room1, self.u_alice, u"test") end = yield self.store.get_room_events_max_id() @@ -125,17 +87,17 @@ class StreamStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_event_stream_get_own(self): # Both bob and alice joins the room - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) # Initial stream key: start = yield self.store.get_room_events_max_id() - yield self.inject_message(self.room1, self.u_alice, u"test") + yield self.event_injector.inject_message(self.room1, self.u_alice, u"test") end = yield self.store.get_room_events_max_id() @@ -162,22 +124,22 @@ class StreamStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_event_stream_join_leave(self): # Both bob and alice joins the room - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) # Then bob leaves again. - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.LEAVE ) # Initial stream key: start = yield self.store.get_room_events_max_id() - yield self.inject_message(self.room1, self.u_alice, u"test") + yield self.event_injector.inject_message(self.room1, self.u_alice, u"test") end = yield self.store.get_room_events_max_id() @@ -193,17 +155,17 @@ class StreamStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_event_stream_prev_content(self): - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) - event1 = yield self.inject_room_member( + event1 = yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) start = yield self.store.get_room_events_max_id() - event2 = yield self.inject_room_member( + event2 = yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN, ) -- cgit 1.5.1 From 6d7f291b93c363ecee7a3f1e5f5e2cdf81a0fbae Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:13:07 +0100 Subject: Front-load spaces --- synapse/config/_base.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index b9983f72a2..ceef309afc 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -27,13 +27,13 @@ class ConfigError(Exception): class Config(object): stats_reporting_begging_spiel = ( - "We would really appreciate it if you could help our project out by " - "reporting anonymized usage statistics from your homeserver. Only very " - "basic aggregate data (e.g. number of users) will be reported, but it " - "helps us to track the growth of the Matrix community, and helps us to " - "make Matrix a success, as well as to convince other networks that they " - "should peer with us.\n" - "Thank you." + "We would really appreciate it if you could help our project out by" + " reporting anonymized usage statistics from your homeserver. Only very" + " basic aggregate data (e.g. number of users) will be reported, but it" + " helps us to track the growth of the Matrix community, and helps us to" + " make Matrix a success, as well as to convince other networks that they" + " should peer with us." + "\nThank you." ) @staticmethod -- cgit 1.5.1 From eb011cd99ba03f40f4ed7a023b64f93dfa2cbdc9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:29:36 +0100 Subject: Add docstring --- synapse/storage/events.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2b51db9940..46df6b4d6d 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -908,6 +908,12 @@ class EventsStore(SQLBaseStore): @defer.inlineCallbacks def count_daily_messages(self): + """ + Returns an estimate of the number of messages sent in the last day. + + If it has been significantly less or more than one day since the last + call to this function, it will return None. + """ def _count_messages(txn): now = self.hs.get_clock().time() -- cgit 1.5.1 From b6e0303c833bb37891294046598d48cafe2b1c9e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:34:29 +0100 Subject: Catch stats-reporting errors --- synapse/app/homeserver.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b4429bd4f3..21840e4a28 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -701,10 +701,13 @@ def run(hs): stats["daily_messages"] = daily_messages logger.info("Reporting stats to matrix.org: %s" % (stats,)) - hs.get_simple_http_client().put_json( - "https://matrix.org/report-usage-stats/push", - stats - ) + try: + yield hs.get_simple_http_client().put_json( + "https://matrix.org/report-usage-stats/push", + stats + ) + except Exception as e: + logger.warn("Error reporting stats: %s", e) if hs.config.report_stats: phone_home_task = task.LoopingCall(phone_stats_home) -- cgit 1.5.1 From 6d59ffe1ce9a821d50d491f97bf05950198f6f53 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:47:40 +0100 Subject: Add some docstrings --- synapse/storage/__init__.py | 3 +++ synapse/storage/registration.py | 1 + 2 files changed, 4 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b64c90d631..340e59afcb 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -128,6 +128,9 @@ class DataStore(RoomMemberStore, RoomStore, @defer.inlineCallbacks def count_daily_users(self): + """ + Counts the number of users who used this homeserver in the last 24 hours. + """ def _count_users(txn): txn.execute( "SELECT COUNT(DISTINCT user_id) AS users" diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 6d76237658..b454dd5b3a 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -292,6 +292,7 @@ class RegistrationStore(SQLBaseStore): @defer.inlineCallbacks def count_all_users(self): + """Counts all users registered on the homeserver.""" def _count_users(txn): txn.execute("SELECT COUNT(*) AS users FROM users") rows = self.cursor_to_dict(txn) -- cgit 1.5.1 From b35baf6f3c5e9f24e9af241eb2423c4b94dd5a14 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 15:13:10 +0100 Subject: Define __repr__ methods for StreamConfig and PaginationConfig So that they can be used with "%r" log formats. --- synapse/streams/config.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/streams/config.py b/synapse/streams/config.py index 2ec7c5403b..167bfe0de3 100644 --- a/synapse/streams/config.py +++ b/synapse/streams/config.py @@ -34,6 +34,11 @@ class SourcePaginationConfig(object): self.direction = 'f' if direction == 'f' else 'b' self.limit = int(limit) if limit is not None else None + def __repr__(self): + return ( + "StreamConfig(from_key=%r, to_key=%r, direction=%r, limit=%r)" + ) % (self.from_key, self.to_key, self.direction, self.limit) + class PaginationConfig(object): @@ -94,10 +99,10 @@ class PaginationConfig(object): logger.exception("Failed to create pagination config") raise SynapseError(400, "Invalid request.") - def __str__(self): + def __repr__(self): return ( - "" + "PaginationConfig(from_tok=%r, to_tok=%r," + " direction=%r, limit=%r)" ) % (self.from_token, self.to_token, self.direction, self.limit) def get_source_config(self, source_name): -- cgit 1.5.1 From cc3ab0c214156cdad9731beb27790e8510b4b023 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:13:06 +0100 Subject: Add dev script for finding where functions are called from, and finding functions that aren't called at all --- scripts-dev/definitions.py | 130 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100755 scripts-dev/definitions.py diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py new file mode 100755 index 0000000000..09cec9d8ec --- /dev/null +++ b/scripts-dev/definitions.py @@ -0,0 +1,130 @@ +#! /usr/bin/python + +import ast +import yaml + +class DefinitionVisitor(ast.NodeVisitor): + def __init__(self): + super(DefinitionVisitor, self).__init__() + self.functions = {} + self.classes = {} + self.names = {} + self.attrs = set() + self.definitions = { + 'def': self.functions, + 'class': self.classes, + 'names': self.names, + 'attrs': self.attrs, + } + + def visit_Name(self, node): + self.names.setdefault(type(node.ctx).__name__, set()).add(node.id) + + def visit_Attribute(self, node): + self.attrs.add(node.attr) + for child in ast.iter_child_nodes(node): + self.visit(child) + + def visit_ClassDef(self, node): + visitor = DefinitionVisitor() + self.classes[node.name] = visitor.definitions + for child in ast.iter_child_nodes(node): + visitor.visit(child) + + def visit_FunctionDef(self, node): + visitor = DefinitionVisitor() + self.functions[node.name] = visitor.definitions + for child in ast.iter_child_nodes(node): + visitor.visit(child) + + +def non_empty(defs): + functions = {name: non_empty(f) for name, f in defs['def'].items()} + classes = {name: non_empty(f) for name, f in defs['class'].items()} + result = {} + if functions: result['def'] = functions + if classes: result['class'] = classes + names = defs['names'] + uses = [] + for name in names.get('Load', ()): + if name not in names.get('Param', ()) and name not in names.get('Store', ()): + uses.append(name) + uses.extend(defs['attrs']) + if uses: result['uses'] = uses + result['names'] = names + result['attrs'] = defs['attrs'] + return result + + +def definitions_in_code(input_code): + input_ast = ast.parse(input_code) + visitor = DefinitionVisitor() + visitor.visit(input_ast) + definitions = non_empty(visitor.definitions) + return definitions + + +def definitions_in_file(filepath): + with open(filepath) as f: + return definitions_in_code(f.read()) + + +def defined_names(prefix, defs, names): + for name, funcs in defs.get('def', {}).items(): + names.setdefault(name, {'defined': []})['defined'].append(prefix + name) + defined_names(prefix + name + ".", funcs, names) + + for name, funcs in defs.get('class', {}).items(): + names.setdefault(name, {'defined': []})['defined'].append(prefix + name) + defined_names(prefix + name + ".", funcs, names) + + +def used_names(prefix, defs, names): + for name, funcs in defs.get('def', {}).items(): + used_names(prefix + name + ".", funcs, names) + + for name, funcs in defs.get('class', {}).items(): + used_names(prefix + name + ".", funcs, names) + + for used in defs.get('uses', ()): + if used in names: + names[used].setdefault('used', []).append(prefix.rstrip('.')) + + +if __name__ == '__main__': + import sys, os + if not sys.argv[1:]: + sys.stderr.write( + "Usage: definitions.py \n" + " definitions.py \n" + "Either list the definitions matching the regexp or list\n" + " 'unused' definitions\n" + ) + + definitions = {} + for root, dirs, files in os.walk(sys.argv[1]): + for filename in files: + if filename.endswith(".py"): + filepath = os.path.join(root, filename) + definitions[filepath] = definitions_in_file(filepath) + + names = {} + for filepath, defs in definitions.items(): + defined_names(filepath + ":", defs, names) + + for filepath, defs in definitions.items(): + used_names(filepath + ":", defs, names) + + if sys.argv[2:]: + import re + pattern = re.compile(sys.argv[2]) + for name in list(names): + if not pattern.match(name): + del names[name] + else: + for name in list(names): + if 'used' in names[name]: + del names[name] + + yaml.dump(names, sys.stdout, default_flow_style=False) + #yaml.dump(definitions, sys.stdout, default_flow_style=False) -- cgit 1.5.1 From 527d95dea0d55bd1932639e61387ef60d834134e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:14:15 +0100 Subject: synapse/storage/_base.py:Table was unused --- synapse/storage/_base.py | 128 ---------------------------------------------- synapse/storage/pusher.py | 4 +- 2 files changed, 2 insertions(+), 130 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 495ef087c9..c1b5423bd6 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -25,8 +25,6 @@ from util.id_generators import IdGenerator, StreamIdGenerator from twisted.internet import defer -from collections import namedtuple - import sys import time import threading @@ -791,129 +789,3 @@ class _RollbackButIsFineException(Exception): something went wrong. """ pass - - -class Table(object): - """ A base class used to store information about a particular table. - """ - - table_name = None - """ str: The name of the table """ - - fields = None - """ list: The field names """ - - EntryType = None - """ Type: A tuple type used to decode the results """ - - _select_where_clause = "SELECT %s FROM %s WHERE %s" - _select_clause = "SELECT %s FROM %s" - _insert_clause = "REPLACE INTO %s (%s) VALUES (%s)" - - @classmethod - def select_statement(cls, where_clause=None): - """ - Args: - where_clause (str): The WHERE clause to use. - - Returns: - str: An SQL statement to select rows from the table with the given - WHERE clause. - """ - if where_clause: - return cls._select_where_clause % ( - ", ".join(cls.fields), - cls.table_name, - where_clause - ) - else: - return cls._select_clause % ( - ", ".join(cls.fields), - cls.table_name, - ) - - @classmethod - def insert_statement(cls): - return cls._insert_clause % ( - cls.table_name, - ", ".join(cls.fields), - ", ".join(["?"] * len(cls.fields)), - ) - - @classmethod - def decode_single_result(cls, results): - """ Given an iterable of tuples, return a single instance of - `EntryType` or None if the iterable is empty - Args: - results (list): The results list to convert to `EntryType` - Returns: - EntryType: An instance of `EntryType` - """ - results = list(results) - if results: - return cls.EntryType(*results[0]) - else: - return None - - @classmethod - def decode_results(cls, results): - """ Given an iterable of tuples, return a list of `EntryType` - Args: - results (list): The results list to convert to `EntryType` - - Returns: - list: A list of `EntryType` - """ - return [cls.EntryType(*row) for row in results] - - @classmethod - def get_fields_string(cls, prefix=None): - if prefix: - to_join = ("%s.%s" % (prefix, f) for f in cls.fields) - else: - to_join = cls.fields - - return ", ".join(to_join) - - -class JoinHelper(object): - """ Used to help do joins on tables by looking at the tables' fields and - creating a list of unique fields to use with SELECTs and a namedtuple - to dump the results into. - - Attributes: - tables (list): List of `Table` classes - EntryType (type) - """ - - def __init__(self, *tables): - self.tables = tables - - res = [] - for table in self.tables: - res += [f for f in table.fields if f not in res] - - self.EntryType = namedtuple("JoinHelperEntry", res) - - def get_fields(self, **prefixes): - """Get a string representing a list of fields for use in SELECT - statements with the given prefixes applied to each. - - For example:: - - JoinHelper(PdusTable, StateTable).get_fields( - PdusTable="pdus", - StateTable="state" - ) - """ - res = [] - for field in self.EntryType._fields: - for table in self.tables: - if field in table.fields: - res.append("%s.%s" % (prefixes[table.__name__], field)) - break - - return ", ".join(res) - - def decode_results(self, rows): - return [self.EntryType(*row) for row in rows] diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 00b748f131..345c4e1104 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, Table +from ._base import SQLBaseStore from twisted.internet import defer from synapse.api.errors import StoreError @@ -149,5 +149,5 @@ class PusherStore(SQLBaseStore): ) -class PushersTable(Table): +class PushersTable(object): table_name = "pushers" -- cgit 1.5.1 From 372ac60375973f03bcfce7f857c85b6ed2388881 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:16:07 +0100 Subject: synapse/util/__init__.py:unwrap_deferred was unused --- synapse/util/__init__.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 07ff25cef3..1d123ccefc 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -29,34 +29,6 @@ def unwrapFirstError(failure): return failure.value.subFailure -def unwrap_deferred(d): - """Given a deferred that we know has completed, return its value or raise - the failure as an exception - """ - if not d.called: - raise RuntimeError("deferred has not finished") - - res = [] - - def f(r): - res.append(r) - return r - d.addCallback(f) - - if res: - return res[0] - - def f(r): - res.append(r) - return r - d.addErrback(f) - - if res: - res[0].raiseException() - else: - raise RuntimeError("deferred did not call callbacks") - - class Clock(object): """A small utility that obtains current time-of-day so that time may be mocked during unit-tests. -- cgit 1.5.1 From f2fcc0a8cff8b11ba0201c2bebea37649960b6a3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:18:45 +0100 Subject: synapse/api/errors.py:RoomError was unused --- synapse/api/errors.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index c3b4d971a8..ee3045268f 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -77,11 +77,6 @@ class SynapseError(CodeMessageException): ) -class RoomError(SynapseError): - """An error raised when a room event fails.""" - pass - - class RegistrationError(SynapseError): """An error raised when a registration event fails.""" pass -- cgit 1.5.1 From a247729806a1cf7093b3c0819094338bf22affa8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:19:49 +0100 Subject: synapse/streams/events.py:StreamSource was unused --- synapse/streams/events.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/synapse/streams/events.py b/synapse/streams/events.py index aaa3609aa5..8671a8fa4e 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -70,15 +70,3 @@ class EventSources(object): ), ) defer.returnValue(token) - - -class StreamSource(object): - def get_new_events_for_user(self, user, from_key, limit): - """from_key is the key within this event source.""" - raise NotImplementedError("get_new_events_for_user") - - def get_current_key(self): - raise NotImplementedError("get_current_key") - - def get_pagination_rows(self, user, pagination_config, key): - raise NotImplementedError("get_rows") -- cgit 1.5.1 From 184ba0968a08e537dccc65d8fe75f65b86e22504 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:25:30 +0100 Subject: synapse/app/homeserver.py:GzipFile was unused --- synapse/app/homeserver.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 21840e4a28..349a4c1774 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -85,12 +85,6 @@ import time logger = logging.getLogger("synapse.app.homeserver") -class GzipFile(File): - def getChild(self, path, request): - child = File.getChild(self, path, request) - return EncodingResourceWrapper(child, [GzipEncoderFactory()]) - - def gz_wrap(r): return EncodingResourceWrapper(r, [GzipEncoderFactory()]) -- cgit 1.5.1 From 7a5818ed811c6da046e60bcc7eacb7a28a92cd73 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:27:22 +0100 Subject: Note that GzipFile was removed in comment that referenced it --- synapse/app/homeserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 349a4c1774..190b03e2f7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -128,6 +128,7 @@ class SynapseHomeServer(HomeServer): # (It can stay enabled for the API resources: they call # write() with the whole body and then finish() straight # after and so do not trigger the bug. + # GzipFile was removed in commit 184ba09 # return GzipFile(webclient_path) # TODO configurable? return File(webclient_path) # TODO configurable? -- cgit 1.5.1 From bb4dddd6c4f85bc5b07119d3f9dec31964b5b6f9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:33:34 +0100 Subject: Move NullSource out of synapse and into tests since it is only used by the tests --- synapse/streams/events.py | 16 ---------------- tests/rest/client/v1/test_presence.py | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/synapse/streams/events.py b/synapse/streams/events.py index 8671a8fa4e..699083ae12 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -23,22 +23,6 @@ from synapse.handlers.typing import TypingNotificationEventSource from synapse.handlers.receipts import ReceiptEventSource -class NullSource(object): - """This event source never yields any events and its token remains at - zero. It may be useful for unit-testing.""" - def __init__(self, hs): - pass - - def get_new_events_for_user(self, user, from_key, limit): - return defer.succeed(([], from_key)) - - def get_current_key(self, direction='f'): - return defer.succeed(0) - - def get_pagination_rows(self, user, pagination_config, key): - return defer.succeed(([], pagination_config.from_key)) - - class EventSources(object): SOURCE_TYPES = { "room": RoomEventSource, diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 2ee3da0b34..29d9bbaad4 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -41,6 +41,22 @@ myid = "@apple:test" PATH_PREFIX = "/_matrix/client/api/v1" +class NullSource(object): + """This event source never yields any events and its token remains at + zero. It may be useful for unit-testing.""" + def __init__(self, hs): + pass + + def get_new_events_for_user(self, user, from_key, limit): + return defer.succeed(([], from_key)) + + def get_current_key(self, direction='f'): + return defer.succeed(0) + + def get_pagination_rows(self, user, pagination_config, key): + return defer.succeed(([], pagination_config.from_key)) + + class JustPresenceHandlers(object): def __init__(self, hs): self.presence_handler = PresenceHandler(hs) @@ -243,7 +259,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): # HIDEOUS HACKERY # TODO(paul): This should be injected in via the HomeServer DI system from synapse.streams.events import ( - PresenceEventSource, NullSource, EventSources + PresenceEventSource, EventSources ) old_SOURCE_TYPES = EventSources.SOURCE_TYPES -- cgit 1.5.1 From 7dd4f79c49e1a1ba4cf2edf8b45ed841a32a33b0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:37:07 +0100 Subject: synapse/storage/_base.py:_execute_and_decode was unused --- synapse/storage/_base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c1b5423bd6..cf4ec30f48 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -374,9 +374,6 @@ class SQLBaseStore(object): return self.runInteraction(desc, interaction) - def _execute_and_decode(self, desc, query, *args): - return self._execute(desc, self.cursor_to_dict, query, *args) - # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. -- cgit 1.5.1 From 3559a835a22be3ae474fc2a14363a59b89ec05b8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:39:46 +0100 Subject: synapse/storage/event_federation.py:_get_auth_events is unused --- synapse/storage/event_federation.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index c1cabbaa60..7ed0d9ae1b 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -225,27 +225,6 @@ class EventFederationStore(SQLBaseStore): return results - def _get_auth_events(self, txn, event_id): - auth_ids = self._simple_select_onecol_txn( - txn, - table="event_auth", - keyvalues={ - "event_id": event_id, - }, - retcol="auth_id", - ) - - results = [] - for auth_id in auth_ids: - hashes = self._get_event_reference_hashes_txn(txn, auth_id) - prev_hashes = { - k: encode_base64(v) for k, v in hashes.items() - if k == "sha256" - } - results.append((auth_id, prev_hashes)) - - return results - def get_min_depth(self, room_id): """ For hte given room, get the minimum depth we have seen for it. """ -- cgit 1.5.1 From 8a519ac76d646c991c4f77973a157ac7bea25665 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 23 Sep 2015 09:55:24 +0100 Subject: Fix demo/start.sh to work with --report-stats --- demo/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/demo/start.sh b/demo/start.sh index 572dbfab0b..a90561488d 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -25,6 +25,7 @@ for port in 8080 8081 8082; do --generate-config \ -H "localhost:$https_port" \ --config-path "$DIR/etc/$port.config" \ + --report-stats no # Check script parameters if [ $# -eq 1 ]; then -- cgit 1.5.1 From 257fa1c53e20b4394ff1493f6112a011c2727e7b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 23 Sep 2015 10:07:31 +0100 Subject: Set m.room.canonical_alias on room creation. --- synapse/handlers/room.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index bb5eef6bbd..e194f39e70 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -155,6 +155,7 @@ class RoomCreationHandler(BaseHandler): preset_config=preset_config, invite_list=invite_list, initial_state=initial_state, + room_alias=room_alias, ) msg_handler = self.hs.get_handlers().message_handler @@ -202,7 +203,7 @@ class RoomCreationHandler(BaseHandler): defer.returnValue(result) def _create_events_for_new_room(self, creator, room_id, preset_config, - invite_list, initial_state): + invite_list, initial_state, room_alias): config = RoomCreationHandler.PRESETS_DICT[preset_config] creator_id = creator.to_string() @@ -271,6 +272,15 @@ class RoomCreationHandler(BaseHandler): returned_events.append(power_levels_event) + if room_alias: + if (EventTypes.CanonicalAlias, '') not in initial_state: + room_alias_event = create( + etype=EventTypes.CanonicalAlias, + content={"alias": room_alias.to_string()}, + ) + + returned_events.append(room_alias_event) + if (EventTypes.JoinRules, '') not in initial_state: join_rules_event = create( etype=EventTypes.JoinRules, -- cgit 1.5.1 From 04abf53a5633d8220b01772aedc573caa0e71f6e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:17:50 +0100 Subject: Use argparse for definition finder --- scripts-dev/definitions.py | 64 +++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py index 09cec9d8ec..934bc4a3d8 100755 --- a/scripts-dev/definitions.py +++ b/scripts-dev/definitions.py @@ -92,21 +92,32 @@ def used_names(prefix, defs, names): if __name__ == '__main__': - import sys, os - if not sys.argv[1:]: - sys.stderr.write( - "Usage: definitions.py \n" - " definitions.py \n" - "Either list the definitions matching the regexp or list\n" - " 'unused' definitions\n" - ) + import sys, os, argparse, re + + parser = argparse.ArgumentParser(description='Find definitions.') + parser.add_argument( + "--unused", action="store_true", help="Only list unused definitions" + ) + parser.add_argument( + "--ignore", action="append", metavar="REGEXP", help="Ignore a pattern" + ) + parser.add_argument( + "--pattern", nargs='+', action="append", metavar="REGEXP", + help="Search for a pattern" + ) + parser.add_argument( + "directories", nargs='+', metavar="DIR", + help="Directories to search for definitions" + ) + args = parser.parse_args() definitions = {} - for root, dirs, files in os.walk(sys.argv[1]): - for filename in files: - if filename.endswith(".py"): - filepath = os.path.join(root, filename) - definitions[filepath] = definitions_in_file(filepath) + for directory in args.directories: + for root, dirs, files in os.walk(directory): + for filename in files: + if filename.endswith(".py"): + filepath = os.path.join(root, filename) + definitions[filepath] = definitions_in_file(filepath) names = {} for filepath, defs in definitions.items(): @@ -115,16 +126,17 @@ if __name__ == '__main__': for filepath, defs in definitions.items(): used_names(filepath + ":", defs, names) - if sys.argv[2:]: - import re - pattern = re.compile(sys.argv[2]) - for name in list(names): - if not pattern.match(name): - del names[name] - else: - for name in list(names): - if 'used' in names[name]: - del names[name] - - yaml.dump(names, sys.stdout, default_flow_style=False) - #yaml.dump(definitions, sys.stdout, default_flow_style=False) + patterns = [re.compile(pattern) for pattern in args.pattern or ()] + ignore = [re.compile(pattern) for pattern in args.ignore or ()] + + result = {} + for name, definition in names.items(): + if patterns and not any(pattern.match(name) for pattern in patterns): + continue + if ignore and any(pattern.match(name) for pattern in ignore): + continue + if args.unused and definition.get('used'): + continue + result[name] = definition + + yaml.dump(result, sys.stdout, default_flow_style=False) -- cgit 1.5.1 From 60728c8c9ebc80c58454639c3f021d43091453ca Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:25:26 +0100 Subject: synapse/handlers/federation.py:_handle_auth_events was unused --- synapse/handlers/federation.py | 49 ------------------------------------------ 1 file changed, 49 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4ff20599d6..f4dce712f9 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1456,52 +1456,3 @@ class FederationHandler(BaseHandler): }, "missing": [e.event_id for e in missing_locals], }) - - @defer.inlineCallbacks - def _handle_auth_events(self, origin, auth_events): - auth_ids_to_deferred = {} - - def process_auth_ev(ev): - auth_ids = [e_id for e_id, _ in ev.auth_events] - - prev_ds = [ - auth_ids_to_deferred[i] - for i in auth_ids - if i in auth_ids_to_deferred - ] - - d = defer.Deferred() - - auth_ids_to_deferred[ev.event_id] = d - - @defer.inlineCallbacks - def f(*_): - ev.internal_metadata.outlier = True - - try: - auth = { - (e.type, e.state_key): e for e in auth_events - if e.event_id in auth_ids - } - - yield self._handle_new_event( - origin, ev, auth_events=auth - ) - except: - logger.exception( - "Failed to handle auth event %s", - ev.event_id, - ) - - d.callback(None) - - if prev_ds: - dx = defer.DeferredList(prev_ds) - dx.addBoth(f) - else: - f() - - for e in auth_events: - process_auth_ev(e) - - yield defer.DeferredList(auth_ids_to_deferred.values()) -- cgit 1.5.1 From 57338a97683de16b325a2e62b4dee4b4a6b77aa2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:26:45 +0100 Subject: synapse/handlers/room.py:_should_invite_join was unused --- synapse/handlers/room.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 243623190f..d60f86bbd0 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -492,32 +492,6 @@ class RoomMemberHandler(BaseHandler): "user_joined_room", user=user, room_id=room_id ) - @defer.inlineCallbacks - def _should_invite_join(self, room_id, prev_state, do_auth): - logger.debug("_should_invite_join: room_id: %s", room_id) - - # XXX: We don't do an auth check if we are doing an invite - # join dance for now, since we're kinda implicitly checking - # that we are allowed to join when we decide whether or not we - # need to do the invite/join dance. - - # Only do an invite join dance if a) we were invited, - # b) the person inviting was from a differnt HS and c) we are - # not currently in the room - room_host = None - if prev_state and prev_state.membership == Membership.INVITE: - room = yield self.store.get_room(room_id) - inviter = UserID.from_string( - prev_state.sender - ) - - is_remote_invite_join = not self.hs.is_mine(inviter) and not room - room_host = inviter.domain - else: - is_remote_invite_join = False - - defer.returnValue((is_remote_invite_join, room_host)) - @defer.inlineCallbacks def get_joined_rooms_for_user(self, user): """Returns a list of roomids that the user has any of the given -- cgit 1.5.1 From 82b8d4b86a981559570fbd1e8ee3dcd38faf553f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:27:47 +0100 Subject: synapse/state.py:_get_state_key_from_event was unused --- synapse/state.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index ed36f844cb..bb225c39cf 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -31,10 +31,6 @@ import hashlib logger = logging.getLogger(__name__) -def _get_state_key_from_event(event): - return event.state_key - - KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) -- cgit 1.5.1 From 1ee3d26432d87ff312350f21da982f646b5af49a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:30:03 +0100 Subject: synapse/storage/_base.py:_simple_selectupdate_one was unused --- synapse/storage/_base.py | 31 ------------------------------- tests/storage/test_base.py | 20 -------------------- 2 files changed, 51 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index cf4ec30f48..79021bde6b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -686,37 +686,6 @@ class SQLBaseStore(object): return dict(zip(retcols, row)) - def _simple_selectupdate_one(self, table, keyvalues, updatevalues=None, - retcols=None, allow_none=False, - desc="_simple_selectupdate_one"): - """ Combined SELECT then UPDATE.""" - def func(txn): - ret = None - if retcols: - ret = self._simple_select_one_txn( - txn, - table=table, - keyvalues=keyvalues, - retcols=retcols, - allow_none=allow_none, - ) - - if updatevalues: - self._simple_update_one_txn( - txn, - table=table, - keyvalues=keyvalues, - updatevalues=updatevalues, - ) - - # if txn.rowcount == 0: - # raise StoreError(404, "No row found") - if txn.rowcount > 1: - raise StoreError(500, "More than one row matched") - - return ret - return self.runInteraction(desc, func) - def _simple_delete_one(self, table, keyvalues, desc="_simple_delete_one"): """Executes a DELETE query on the named table, expecting to delete a single row. diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 8573f18b55..1ddca1da4c 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -185,26 +185,6 @@ class SQLBaseStoreTestCase(unittest.TestCase): [3, 4, 1, 2] ) - @defer.inlineCallbacks - def test_update_one_with_return(self): - self.mock_txn.rowcount = 1 - self.mock_txn.fetchone.return_value = ("Old Value",) - - ret = yield self.datastore._simple_selectupdate_one( - table="tablename", - keyvalues={"keycol": "TheKey"}, - updatevalues={"columname": "New Value"}, - retcols=["columname"] - ) - - self.assertEquals({"columname": "Old Value"}, ret) - self.mock_txn.execute.assert_has_calls([ - call('SELECT columname FROM tablename WHERE keycol = ?', - ['TheKey']), - call("UPDATE tablename SET columname = ? WHERE keycol = ?", - ["New Value", "TheKey"]) - ]) - @defer.inlineCallbacks def test_delete_one(self): self.mock_txn.rowcount = 1 -- cgit 1.5.1 From 1d9036aff2be9ce089114b1389cc86a34ccd8490 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:30:25 +0100 Subject: synapse/storage/_base.py:_simple_delete was unused --- synapse/storage/_base.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 79021bde6b..33751f3092 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -707,16 +707,6 @@ class SQLBaseStore(object): raise StoreError(500, "more than one row matched") return self.runInteraction(desc, func) - def _simple_delete(self, table, keyvalues, desc="_simple_delete"): - """Executes a DELETE query on the named table. - - Args: - table : string giving the table name - keyvalues : dict of column names and values to select the row with - """ - - return self.runInteraction(desc, self._simple_delete_txn) - def _simple_delete_txn(self, txn, table, keyvalues): sql = "DELETE FROM %s WHERE %s" % ( table, -- cgit 1.5.1 From 396834f1c010899cae87aac45143bbd9911adb4f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:30:38 +0100 Subject: synapse/storage/_base.py:_simple_max_id was unused --- synapse/storage/_base.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 33751f3092..693784ad38 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -715,24 +715,6 @@ class SQLBaseStore(object): return txn.execute(sql, keyvalues.values()) - def _simple_max_id(self, table): - """Executes a SELECT query on the named table, expecting to return the - max value for the column "id". - - Args: - table : string giving the table name - """ - sql = "SELECT MAX(id) AS id FROM %s" % table - - def func(txn): - txn.execute(sql) - max_id = self.cursor_to_dict(txn)[0]["id"] - if max_id is None: - return 0 - return max_id - - return self.runInteraction("_simple_max_id", func) - def get_next_stream_id(self): with self._next_stream_id_lock: i = self._next_stream_id -- cgit 1.5.1 From c292dba70cca26751cdab90a6e50200828959642 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:31:25 +0100 Subject: Remove unused functions from synapse/storage/event_federation.py --- synapse/storage/event_federation.py | 71 ------------------------------------- 1 file changed, 71 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 7ed0d9ae1b..6d4421dd8f 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -154,77 +154,6 @@ class EventFederationStore(SQLBaseStore): return results - def _get_latest_state_in_room(self, txn, room_id, type, state_key): - event_ids = self._simple_select_onecol_txn( - txn, - table="state_forward_extremities", - keyvalues={ - "room_id": room_id, - "type": type, - "state_key": state_key, - }, - retcol="event_id", - ) - - results = [] - for event_id in event_ids: - hashes = self._get_event_reference_hashes_txn(txn, event_id) - prev_hashes = { - k: encode_base64(v) for k, v in hashes.items() - if k == "sha256" - } - results.append((event_id, prev_hashes)) - - return results - - def _get_prev_events(self, txn, event_id): - results = self._get_prev_events_and_state( - txn, - event_id, - is_state=0, - ) - - return [(e_id, h, ) for e_id, h, _ in results] - - def _get_prev_state(self, txn, event_id): - results = self._get_prev_events_and_state( - txn, - event_id, - is_state=True, - ) - - return [(e_id, h, ) for e_id, h, _ in results] - - def _get_prev_events_and_state(self, txn, event_id, is_state=None): - keyvalues = { - "event_id": event_id, - } - - if is_state is not None: - keyvalues["is_state"] = bool(is_state) - - res = self._simple_select_list_txn( - txn, - table="event_edges", - keyvalues=keyvalues, - retcols=["prev_event_id", "is_state"], - ) - - hashes = self._get_prev_event_hashes_txn(txn, event_id) - - results = [] - for d in res: - edge_hash = self._get_event_reference_hashes_txn(txn, d["prev_event_id"]) - edge_hash.update(hashes.get(d["prev_event_id"], {})) - prev_hashes = { - k: encode_base64(v) - for k, v in edge_hash.items() - if k == "sha256" - } - results.append((d["prev_event_id"], prev_hashes, d["is_state"])) - - return results - def get_min_depth(self, room_id): """ For hte given room, get the minimum depth we have seen for it. """ -- cgit 1.5.1 From 92d8d724c5c34f0a83cbd8c5dce7f0c0c21a1568 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:33:06 +0100 Subject: Remove unused functions from synapse/storage/events.py --- synapse/storage/events.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 46df6b4d6d..416ef6af93 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -890,22 +890,11 @@ class EventsStore(SQLBaseStore): return ev - def _parse_events(self, rows): - return self.runInteraction( - "_parse_events", self._parse_events_txn, rows - ) - def _parse_events_txn(self, txn, rows): event_ids = [r["event_id"] for r in rows] return self._get_events_txn(txn, event_ids) - def _has_been_redacted_txn(self, txn, event): - sql = "SELECT event_id FROM redactions WHERE redacts = ?" - txn.execute(sql, (event.event_id,)) - result = txn.fetchone() - return result[0] if result else None - @defer.inlineCallbacks def count_daily_messages(self): """ -- cgit 1.5.1 From e51aa4be9691e7c6918d72810ee0b751f2e48797 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:35:10 +0100 Subject: synapse/storage/roommember.py:_get_members_query was unused --- synapse/storage/roommember.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index cd9eefbd9f..e17cbe677a 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -178,12 +178,6 @@ class RoomMemberStore(SQLBaseStore): return joined_domains - def _get_members_query(self, where_clause, where_values): - return self.runInteraction( - "get_members_query", self._get_members_events_txn, - where_clause, where_values - ).addCallbacks(self._get_events) - def _get_members_events_txn(self, txn, room_id, membership=None, user_id=None): rows = self._get_members_rows_txn( txn, -- cgit 1.5.1 From 973ebb66bacfeece1f88f8ea71b86186f9c9163e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:36:33 +0100 Subject: Remove unused functions from synapse/storage/signatures.py --- synapse/storage/signatures.py | 112 ------------------------------------------ 1 file changed, 112 deletions(-) diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index ab57b92174..b070be504d 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -24,41 +24,6 @@ from synapse.crypto.event_signing import compute_event_reference_hash class SignatureStore(SQLBaseStore): """Persistence for event signatures and hashes""" - def _get_event_content_hashes_txn(self, txn, event_id): - """Get all the hashes for a given Event. - Args: - txn (cursor): - event_id (str): Id for the Event. - Returns: - A dict of algorithm -> hash. - """ - query = ( - "SELECT algorithm, hash" - " FROM event_content_hashes" - " WHERE event_id = ?" - ) - txn.execute(query, (event_id, )) - return dict(txn.fetchall()) - - def _store_event_content_hash_txn(self, txn, event_id, algorithm, - hash_bytes): - """Store a hash for a Event - Args: - txn (cursor): - event_id (str): Id for the Event. - algorithm (str): Hashing algorithm. - hash_bytes (bytes): Hash function output bytes. - """ - self._simple_insert_txn( - txn, - "event_content_hashes", - { - "event_id": event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }, - ) - def get_event_reference_hashes(self, event_ids): def f(txn): return [ @@ -123,80 +88,3 @@ class SignatureStore(SQLBaseStore): table="event_reference_hashes", values=vals, ) - - def _get_event_signatures_txn(self, txn, event_id): - """Get all the signatures for a given PDU. - Args: - txn (cursor): - event_id (str): Id for the Event. - Returns: - A dict of sig name -> dict(key_id -> signature_bytes) - """ - query = ( - "SELECT signature_name, key_id, signature" - " FROM event_signatures" - " WHERE event_id = ? " - ) - txn.execute(query, (event_id, )) - rows = txn.fetchall() - - res = {} - - for name, key, sig in rows: - res.setdefault(name, {})[key] = sig - - return res - - def _store_event_signature_txn(self, txn, event_id, signature_name, key_id, - signature_bytes): - """Store a signature from the origin server for a PDU. - Args: - txn (cursor): - event_id (str): Id for the Event. - origin (str): origin of the Event. - key_id (str): Id for the signing key. - signature (bytes): The signature. - """ - self._simple_insert_txn( - txn, - "event_signatures", - { - "event_id": event_id, - "signature_name": signature_name, - "key_id": key_id, - "signature": buffer(signature_bytes), - }, - ) - - def _get_prev_event_hashes_txn(self, txn, event_id): - """Get all the hashes for previous PDUs of a PDU - Args: - txn (cursor): - event_id (str): Id for the Event. - Returns: - dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes. - """ - query = ( - "SELECT prev_event_id, algorithm, hash" - " FROM event_edge_hashes" - " WHERE event_id = ?" - ) - txn.execute(query, (event_id, )) - results = {} - for prev_event_id, algorithm, hash_bytes in txn.fetchall(): - hashes = results.setdefault(prev_event_id, {}) - hashes[algorithm] = hash_bytes - return results - - def _store_prev_event_hash_txn(self, txn, event_id, prev_event_id, - algorithm, hash_bytes): - self._simple_insert_txn( - txn, - "event_edge_hashes", - { - "event_id": event_id, - "prev_event_id": prev_event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }, - ) -- cgit 1.5.1 From 1cd65a8d1e9fe30506aa7f19c60e6271a24e6ae9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:37:58 +0100 Subject: synapse/storage/state.py: _make_group_id was unused --- synapse/storage/state.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 9630efcfcc..e935b9443b 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -20,8 +20,6 @@ from synapse.util.caches.descriptors import ( from twisted.internet import defer -from synapse.util.stringutils import random_string - import logging logger = logging.getLogger(__name__) @@ -428,7 +426,3 @@ class StateStore(SQLBaseStore): } defer.returnValue(results) - - -def _make_group_id(clock): - return str(int(clock.time_msec())) + random_string(5) -- cgit 1.5.1 From 7d55314277e29afb25da9c0910b8b0422d422671 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:42:02 +0100 Subject: Remove unused _execute_and_decode from scripts/synapse_port_db --- scripts/synapse_port_db | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 6aba72e459..62515997b1 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -95,8 +95,6 @@ class Store(object): _simple_update_one = SQLBaseStore.__dict__["_simple_update_one"] _simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"] - _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"] - def runInteraction(self, desc, func, *args, **kwargs): def r(conn): try: -- cgit 1.5.1 From 314aabba82d382d36f9ac91050e77deb6018fbcd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:45:33 +0100 Subject: Fix scripts-dev/definitions.py argparse options --- scripts-dev/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py index 934bc4a3d8..f0d0cd8a3f 100755 --- a/scripts-dev/definitions.py +++ b/scripts-dev/definitions.py @@ -102,7 +102,7 @@ if __name__ == '__main__': "--ignore", action="append", metavar="REGEXP", help="Ignore a pattern" ) parser.add_argument( - "--pattern", nargs='+', action="append", metavar="REGEXP", + "--pattern", action="append", metavar="REGEXP", help="Search for a pattern" ) parser.add_argument( -- cgit 1.5.1 From cf1100887b454535e25dfeb67d649c2a4673eab7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 17:34:02 +0100 Subject: Fix order of ON constraints in _get_rooms_for_user_where_membership_is_txn --- synapse/storage/roommember.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index cd9eefbd9f..41c939efb1 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -142,12 +142,12 @@ class RoomMemberStore(SQLBaseStore): sql = ( "SELECT m.room_id, m.sender, m.membership, m.event_id, e.stream_ordering" - " FROM room_memberships as m" - " INNER JOIN current_state_events as c" - " ON e.event_id = c.event_id " - " INNER JOIN events as e " - " ON m.event_id = c.event_id " - " AND m.room_id = c.room_id " + " FROM current_state_events as c" + " INNER JOIN room_memberships as m" + " ON m.event_id = c.event_id" + " INNER JOIN events as e" + " ON e.event_id = c.event_id" + " AND m.room_id = c.room_id" " AND m.user_id = c.state_key" " WHERE %s" ) % (where_clause,) -- cgit 1.5.1 From 76328b85f66f03908a36904102b6b8437fa41663 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 24 Sep 2015 21:50:20 +0000 Subject: Allow config file path to be configurable in in synctl Also, allow it to be run from directories other than the synapse directory --- synapse/app/synctl.py | 50 ++++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 6bcc437591..af4f9ff7e0 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -16,38 +16,25 @@ import sys import os +import os.path import subprocess import signal import yaml SYNAPSE = ["python", "-B", "-m", "synapse.app.homeserver"] -CONFIGFILE = "homeserver.yaml" - GREEN = "\x1b[1;32m" RED = "\x1b[1;31m" NORMAL = "\x1b[m" -if not os.path.exists(CONFIGFILE): - sys.stderr.write( - "No config file found\n" - "To generate a config file, run '%s -c %s --generate-config" - " --server-name='\n" % ( - " ".join(SYNAPSE), CONFIGFILE - ) - ) - sys.exit(1) - -CONFIG = yaml.load(open(CONFIGFILE)) -PIDFILE = CONFIG["pid_file"] - -def start(): +def start(configfile): print "Starting ...", args = SYNAPSE - args.extend(["--daemonize", "-c", CONFIGFILE]) + args.extend(["--daemonize", "-c", configfile]) + cwd = os.path.dirname(os.path.abspath(__file__)) try: - subprocess.check_call(args) + subprocess.check_call(args, cwd=cwd) print GREEN + "started" + NORMAL except subprocess.CalledProcessError as e: print ( @@ -57,24 +44,39 @@ def start(): ) -def stop(): - if os.path.exists(PIDFILE): - pid = int(open(PIDFILE).read()) +def stop(pidfile): + if os.path.exists(pidfile): + pid = int(open(pidfile).read()) os.kill(pid, signal.SIGTERM) print GREEN + "stopped" + NORMAL def main(): + configfile = sys.argv[2] if len(sys.argv) == 3 else "homeserver.yaml" + + if not os.path.exists(configfile): + sys.stderr.write( + "No config file found\n" + "To generate a config file, run '%s -c %s --generate-config" + " --server-name='\n" % ( + " ".join(SYNAPSE), configfile + ) + ) + sys.exit(1) + + config = yaml.load(open(configfile)) + pidfile = config["pid_file"] + action = sys.argv[1] if sys.argv[1:] else "usage" if action == "start": - start() + start(configfile) elif action == "stop": - stop() + stop(pidfile) elif action == "restart": stop() start() else: - sys.stderr.write("Usage: %s [start|stop|restart]\n" % (sys.argv[0],)) + sys.stderr.write("Usage: %s [start|stop|restart] [configfile]\n" % (sys.argv[0],)) sys.exit(1) -- cgit 1.5.1 From f87a11e0fdf4c1a6ebca1ab3f5e8cf33ac34c2b4 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 24 Sep 2015 21:59:38 +0000 Subject: Fix restart --- synapse/app/synctl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index af4f9ff7e0..1078d19b79 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -73,8 +73,8 @@ def main(): elif action == "stop": stop(pidfile) elif action == "restart": - stop() - start() + stop(pidfile) + start(configfile) else: sys.stderr.write("Usage: %s [start|stop|restart] [configfile]\n" % (sys.argv[0],)) sys.exit(1) -- cgit 1.5.1 From 9d39615b7d4d5525ab814d2d84e7f2b4523d0417 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Sep 2015 16:37:59 +0100 Subject: Rename var --- synapse/handlers/federation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e79a82cfc0..3ce1aee52c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1033,15 +1033,15 @@ class FederationHandler(BaseHandler): break for e in itertools.chain(auth_events, state, [event]): - a = { + auth_for_e = { (event_map[e_id].type, event_map[e_id].state_key): event_map[e_id] for e_id, _ in e.auth_events } if create_event: - a[(EventTypes.Create, "")] = create_event + auth_for_e[(EventTypes.Create, "")] = create_event try: - self.auth.check(e, auth_events=a) + self.auth.check(e, auth_events=auth_for_e) except AuthError as err: logger.warn( "Rejecting %s because %s", -- cgit 1.5.1 From 83892d0d3039965ae3075df166cbdbd7339cb0bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Sep 2015 16:41:48 +0100 Subject: Comment --- synapse/handlers/federation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3ce1aee52c..17f4ddd325 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1013,6 +1013,14 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _persist_auth_tree(self, auth_events, state, event): + """Checks the auth chain is valid (and passes auth checks) for the + state and event. Then persists the auth chain and state atomically. + Persists the event seperately. + + Returns: + 2-tuple of (event_stream_id, max_stream_id) from the persist_event + call for `event` + """ events_to_context = {} for e in itertools.chain(auth_events, state): ctx = yield self.state_handler.compute_event_context( -- cgit 1.5.1 From ecd0c0dfc50ceed16aa47cf066bc412211af2335 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Sep 2015 16:46:24 +0100 Subject: Remove double indentation --- synapse/handlers/room.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index e194f39e70..2b15136bda 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -272,14 +272,13 @@ class RoomCreationHandler(BaseHandler): returned_events.append(power_levels_event) - if room_alias: - if (EventTypes.CanonicalAlias, '') not in initial_state: - room_alias_event = create( - etype=EventTypes.CanonicalAlias, - content={"alias": room_alias.to_string()}, - ) + if room_alias and (EventTypes.CanonicalAlias, '') not in initial_state: + room_alias_event = create( + etype=EventTypes.CanonicalAlias, + content={"alias": room_alias.to_string()}, + ) - returned_events.append(room_alias_event) + returned_events.append(room_alias_event) if (EventTypes.JoinRules, '') not in initial_state: join_rules_event = create( -- cgit 1.5.1 From 0a4b7226fc0ce163c7ba2a1a62d6125b3fd1e55d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Oct 2015 09:21:27 +0100 Subject: Don't change cwd in synctl --- synapse/app/synctl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 1078d19b79..5d82beed0e 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -32,9 +32,9 @@ def start(configfile): print "Starting ...", args = SYNAPSE args.extend(["--daemonize", "-c", configfile]) - cwd = os.path.dirname(os.path.abspath(__file__)) + try: - subprocess.check_call(args, cwd=cwd) + subprocess.check_call(args) print GREEN + "started" + NORMAL except subprocess.CalledProcessError as e: print ( -- cgit 1.5.1 From bad780a19705cbffcdd181d3ffc81f10980ed109 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 14:01:52 +0100 Subject: Validate the receipt type before passing it on to the receipt handler --- synapse/rest/client/v2_alpha/receipts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/rest/client/v2_alpha/receipts.py b/synapse/rest/client/v2_alpha/receipts.py index 52e99f54d5..b107b7ce17 100644 --- a/synapse/rest/client/v2_alpha/receipts.py +++ b/synapse/rest/client/v2_alpha/receipts.py @@ -15,6 +15,7 @@ from twisted.internet import defer +from synapse.api.errors import SynapseError from synapse.http.servlet import RestServlet from ._base import client_v2_pattern @@ -41,6 +42,9 @@ class ReceiptRestServlet(RestServlet): def on_POST(self, request, room_id, receipt_type, event_id): user, _ = yield self.auth.get_user_by_req(request) + if receipt_type != "m.read": + raise SynapseError(400, "Receipt type must be 'm.read'") + yield self.receipts_handler.received_client_receipt( room_id, receipt_type, -- cgit 1.5.1 From 5b3e9713dd098df95b321f216105b2298deaeb92 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 1 Oct 2015 17:49:52 +0100 Subject: Implement third party identifier invites --- synapse/api/auth.py | 33 +++++++- synapse/api/constants.py | 1 + synapse/federation/federation_client.py | 9 ++- synapse/federation/federation_server.py | 19 ++++- synapse/federation/transport/client.py | 5 +- synapse/federation/transport/server.py | 2 +- synapse/handlers/_base.py | 11 +++ synapse/handlers/federation.py | 16 +++- synapse/handlers/room.py | 4 + synapse/rest/client/v1/room.py | 132 ++++++++++++++++++++++++++++++-- synapse/util/thirdpartyinvites.py | 62 +++++++++++++++ 11 files changed, 275 insertions(+), 19 deletions(-) create mode 100644 synapse/util/thirdpartyinvites.py diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 847ff60671..37f7f1bf79 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -14,15 +14,19 @@ # limitations under the License. """This module contains classes for authenticating the user.""" +from nacl.exceptions import BadSignatureError from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError from synapse.util.logutils import log_function +from synapse.util.thirdpartyinvites import ThirdPartyInvites from synapse.types import UserID, EventID +from unpaddedbase64 import decode_base64 import logging +import nacl.signing import pymacaroons logger = logging.getLogger(__name__) @@ -31,6 +35,7 @@ logger = logging.getLogger(__name__) AuthEventTypes = ( EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels, EventTypes.JoinRules, EventTypes.RoomHistoryVisibility, + EventTypes.ThirdPartyInvite, ) @@ -318,7 +323,8 @@ class Auth(object): pass elif join_rule == JoinRules.INVITE: if not caller_in_room and not caller_invited: - raise AuthError(403, "You are not invited to this room.") + if not self._verify_third_party_invite(event, auth_events): + raise AuthError(403, "You are not invited to this room.") else: # TODO (erikj): may_join list # TODO (erikj): private rooms @@ -344,6 +350,31 @@ class Auth(object): return True + def _verify_third_party_invite(self, event, auth_events): + for key in ThirdPartyInvites.JOIN_KEYS: + if key not in event.content: + return False + token = event.content["token"] + invite_event = auth_events.get( + (EventTypes.ThirdPartyInvite, token,) + ) + if not invite_event: + return False + try: + public_key = event.content["public_key"] + key_validity_url = event.content["key_validity_url"] + if invite_event.content["public_key"] != public_key: + return False + if invite_event.content["key_validity_url"] != key_validity_url: + return False + verify_key = nacl.signing.VerifyKey(decode_base64(public_key)) + encoded_signature = event.content["signature"] + signature = decode_base64(encoded_signature) + verify_key.verify(token, signature) + return True + except (KeyError, BadSignatureError,): + return False + def _get_power_level_event(self, auth_events): key = (EventTypes.PowerLevels, "", ) return auth_events.get(key) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 3385664394..bfc230d126 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -63,6 +63,7 @@ class EventTypes(object): PowerLevels = "m.room.power_levels" Aliases = "m.room.aliases" Redaction = "m.room.redaction" + ThirdPartyInvite = "m.room.third_party_invite" RoomHistoryVisibility = "m.room.history_visibility" CanonicalAlias = "m.room.canonical_alias" diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index f5e346cdbc..bf22913d4f 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -25,6 +25,7 @@ from synapse.api.errors import ( from synapse.util import unwrapFirstError from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.logutils import log_function +from synapse.util.thirdpartyinvites import ThirdPartyInvites from synapse.events import FrozenEvent import synapse.metrics @@ -356,18 +357,22 @@ class FederationClient(FederationBase): defer.returnValue(signed_auth) @defer.inlineCallbacks - def make_join(self, destinations, room_id, user_id): + def make_join(self, destinations, room_id, user_id, content): for destination in destinations: if destination == self.server_name: continue + args = {} + if ThirdPartyInvites.has_join_keys(content): + ThirdPartyInvites.copy_join_keys(content, args) try: ret = yield self.transport_layer.make_join( - destination, room_id, user_id + destination, room_id, user_id, args ) pdu_dict = ret["event"] + logger.debug("Got response to make_join: %s", pdu_dict) defer.returnValue( diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 725c6f3fa5..d71ab44271 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -23,10 +23,12 @@ from synapse.util.logutils import log_function from synapse.events import FrozenEvent import synapse.metrics -from synapse.api.errors import FederationError, SynapseError +from synapse.api.errors import FederationError, SynapseError, Codes from synapse.crypto.event_signing import compute_event_signature +from synapse.util.thirdpartyinvites import ThirdPartyInvites + import simplejson as json import logging @@ -228,8 +230,19 @@ class FederationServer(FederationBase): ) @defer.inlineCallbacks - def on_make_join_request(self, room_id, user_id): - pdu = yield self.handler.on_make_join_request(room_id, user_id) + def on_make_join_request(self, room_id, user_id, query): + threepid_details = {} + if ThirdPartyInvites.has_join_keys(query): + for k in ThirdPartyInvites.JOIN_KEYS: + if not isinstance(query[k], list) or len(query[k]) != 1: + raise FederationError( + "FATAL", + Codes.MISSING_PARAM, + "key %s value %s" % (k, query[k],), + None + ) + threepid_details[k] = query[k][0] + pdu = yield self.handler.on_make_join_request(room_id, user_id, threepid_details) time_now = self._clock.time_msec() defer.returnValue({"event": pdu.get_pdu_json(time_now)}) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index ced703364b..ae4195e83a 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -160,13 +160,14 @@ class TransportLayerClient(object): @defer.inlineCallbacks @log_function - def make_join(self, destination, room_id, user_id, retry_on_dns_fail=True): + def make_join(self, destination, room_id, user_id, args={}): path = PREFIX + "/make_join/%s/%s" % (room_id, user_id) content = yield self.client.get_json( destination=destination, path=path, - retry_on_dns_fail=retry_on_dns_fail, + args=args, + retry_on_dns_fail=True, ) defer.returnValue(content) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 36f250e1a3..6e394f039e 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -292,7 +292,7 @@ class FederationMakeJoinServlet(BaseFederationServlet): @defer.inlineCallbacks def on_GET(self, origin, content, query, context, user_id): - content = yield self.handler.on_make_join_request(context, user_id) + content = yield self.handler.on_make_join_request(context, user_id, query) defer.returnValue((200, content)) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 60ac6617ae..52434920e3 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -21,6 +21,7 @@ from synapse.api.constants import Membership, EventTypes from synapse.types import UserID, RoomAlias from synapse.util.logcontext import PreserveLoggingContext +from synapse.util.thirdpartyinvites import ThirdPartyInvites import logging @@ -123,6 +124,16 @@ class BaseHandler(object): ) ) + if ( + event.type == EventTypes.Member and + event.content["membership"] == Membership.JOIN and + ThirdPartyInvites.has_join_keys(event.content) + ): + yield ThirdPartyInvites.check_key_valid( + self.hs.get_simple_http_client(), + event + ) + (event_stream_id, max_stream_id) = yield self.store.persist_event( event, context=context ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f4dce712f9..d3d172b7b4 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -39,7 +39,7 @@ from twisted.internet import defer import itertools import logging - +from synapse.util.thirdpartyinvites import ThirdPartyInvites logger = logging.getLogger(__name__) @@ -572,7 +572,8 @@ class FederationHandler(BaseHandler): origin, pdu = yield self.replication_layer.make_join( target_hosts, room_id, - joinee + joinee, + content ) logger.debug("Got response to make_join: %s", pdu) @@ -712,14 +713,18 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function - def on_make_join_request(self, room_id, user_id): + def on_make_join_request(self, room_id, user_id, query): """ We've received a /make_join/ request, so we create a partial join event for the room and return that. We don *not* persist or process it until the other server has signed it and sent it back. """ + event_content = {"membership": Membership.JOIN} + if ThirdPartyInvites.has_join_keys(query): + ThirdPartyInvites.copy_join_keys(query, event_content) + builder = self.event_builder_factory.new({ "type": EventTypes.Member, - "content": {"membership": Membership.JOIN}, + "content": event_content, "room_id": room_id, "sender": user_id, "state_key": user_id, @@ -731,6 +736,9 @@ class FederationHandler(BaseHandler): self.auth.check(event, auth_events=context.current_state) + if ThirdPartyInvites.has_join_keys(event.content): + ThirdPartyInvites.check_key_valid(self.hs.get_simple_http_client(), event) + defer.returnValue(event) @defer.inlineCallbacks diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 773f0a2e92..1c79bc194a 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -462,6 +462,10 @@ class RoomMemberHandler(BaseHandler): if prev_state and prev_state.membership == Membership.INVITE: inviter = UserID.from_string(prev_state.user_id) + should_do_dance = not self.hs.is_mine(inviter) + room_hosts = [inviter.domain] + elif "sender" in event.content: + inviter = UserID.from_string(event.content["sender"]) should_do_dance = not self.hs.is_mine(inviter) room_hosts = [inviter.domain] else: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 23871f161e..ba37061290 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -17,7 +17,7 @@ from twisted.internet import defer from base import ClientV1RestServlet, client_path_pattern -from synapse.api.errors import SynapseError, Codes +from synapse.api.errors import SynapseError, Codes, AuthError from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID, RoomAlias @@ -26,7 +26,7 @@ from synapse.events.utils import serialize_event import simplejson as json import logging import urllib - +from synapse.util.thirdpartyinvites import ThirdPartyInvites logger = logging.getLogger(__name__) @@ -415,9 +415,35 @@ class RoomMembershipRestServlet(ClientV1RestServlet): # target user is you unless it is an invite state_key = user.to_string() if membership_action in ["invite", "ban", "kick"]: - if "user_id" not in content: - raise SynapseError(400, "Missing user_id key.") - state_key = content["user_id"] + try: + state_key = content["user_id"] + except KeyError: + if ( + membership_action != "invite" or + not ThirdPartyInvites.has_invite_keys(content) + ): + raise SynapseError(400, "Missing user_id key.") + + + id_server = content["id_server"] + medium = content["medium"] + address = content["address"] + display_name = content["display_name"] + state_key = yield self._lookup_3pid_user(id_server, medium, address) + if not state_key: + yield self._make_and_store_3pid_invite( + id_server, + display_name, + medium, + address, + room_id, + user, + token_id, + txn_id=txn_id + ) + defer.returnValue((200, {})) + return + # make sure it looks like a user ID; it'll throw if it's invalid. UserID.from_string(state_key) @@ -425,10 +451,18 @@ class RoomMembershipRestServlet(ClientV1RestServlet): membership_action = "leave" msg_handler = self.handlers.message_handler + + event_content = { + "membership": unicode(membership_action), + } + + if membership_action == "join" and ThirdPartyInvites.has_join_keys(content): + ThirdPartyInvites.copy_join_keys(content, event_content) + yield msg_handler.create_and_send_event( { "type": EventTypes.Member, - "content": {"membership": unicode(membership_action)}, + "content": event_content, "room_id": room_id, "sender": user.to_string(), "state_key": state_key, @@ -439,6 +473,92 @@ class RoomMembershipRestServlet(ClientV1RestServlet): defer.returnValue((200, {})) + @defer.inlineCallbacks + def _lookup_3pid_user(self, id_server, medium, address): + """Looks up a 3pid in the passed identity server. + + Args: + id_server (str): The server name (including port, if required) + of the identity server to use. + medium (str): The type of the third party identifier (e.g. "email"). + address (str): The third party identifier (e.g. "foo@example.com"). + + Returns: + (str) the matrix ID of the 3pid, or None if it is not recognized. + """ + try: + data = yield self.hs.get_simple_http_client().get_json( + "https://%s/_matrix/identity/api/v1/lookup" % (id_server,), + { + "medium": medium, + "address": address, + } + ) + + if "mxid" in data: + # TODO: Validate the response signature and such + defer.returnValue(data["mxid"]) + except IOError: + # TODO: Log something maybe? + defer.returnValue(None) + + @defer.inlineCallbacks + def _make_and_store_3pid_invite( + self, + id_server, + display_name, + medium, + address, + room_id, + user, + token_id, + txn_id + ): + token, public_key, key_validity_url = ( + yield self._ask_id_server_for_third_party_invite( + id_server, + medium, + address, + room_id, + user.to_string() + ) + ) + msg_handler = self.handlers.message_handler + yield msg_handler.create_and_send_event( + { + "type": EventTypes.ThirdPartyInvite, + "content": { + "display_name": display_name, + "key_validity_url": key_validity_url, + "public_key": public_key, + }, + "room_id": room_id, + "sender": user.to_string(), + "state_key": token, + }, + token_id=token_id, + txn_id=txn_id, + ) + + @defer.inlineCallbacks + def _ask_id_server_for_third_party_invite( + self, id_server, medium, address, room_id, sender): + is_url = "https://%s/_matrix/identity/api/v1/nonce-it-up" % (id_server,) + data = yield self.hs.get_simple_http_client().post_urlencoded_get_json( + is_url, + { + "medium": medium, + "address": address, + "room_id": room_id, + "sender": sender, + } + ) + # TODO: Check for success + token = data["token"] + public_key = data["public_key"] + key_validity_url = "https://%s/_matrix/identity/api/v1/pubkey/isvalid" % (id_server,) + defer.returnValue((token, public_key, key_validity_url)) + @defer.inlineCallbacks def on_PUT(self, request, room_id, membership_action, txn_id): try: diff --git a/synapse/util/thirdpartyinvites.py b/synapse/util/thirdpartyinvites.py new file mode 100644 index 0000000000..c30279de67 --- /dev/null +++ b/synapse/util/thirdpartyinvites.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer +from synapse.api.errors import AuthError + + +class ThirdPartyInvites(object): + INVITE_KEYS = {"id_server", "medium", "address", "display_name"} + + JOIN_KEYS = { + "token", + "public_key", + "key_validity_url", + "signature", + "sender", + } + + @classmethod + def has_invite_keys(cls, content): + for key in cls.INVITE_KEYS: + if key not in content: + return False + return True + + @classmethod + def has_join_keys(cls, content): + for key in cls.JOIN_KEYS: + if key not in content: + return False + return True + + @classmethod + def copy_join_keys(cls, src, dst): + for key in cls.JOIN_KEYS: + if key in src: + dst[key] = src[key] + + @classmethod + @defer.inlineCallbacks + def check_key_valid(cls, http_client, event): + try: + response = yield http_client.get_json( + event.content["key_validity_url"], + {"public_key": event.content["public_key"]} + ) + if not response["valid"]: + raise AuthError(403, "Third party certificate was invalid") + except IOError: + raise AuthError(403, "Third party certificate could not be checked") -- cgit 1.5.1 From f31014b18f618d81cb667c2b01146b246d32760c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 1 Oct 2015 17:53:07 +0100 Subject: Start updating the sync API to match the specification --- synapse/api/filtering.py | 5 ++- synapse/handlers/sync.py | 64 ++++++++++++++---------------- synapse/rest/client/v2_alpha/sync.py | 75 +++++++++++------------------------- 3 files changed, 54 insertions(+), 90 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 4d570b74f8..c066ce89ef 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -54,7 +54,7 @@ class Filtering(object): ] room_level_definitions = [ - "state", "events", "ephemeral" + "state", "timeline", "ephemeral" ] for key in top_level_definitions: @@ -135,6 +135,9 @@ class Filter(object): def __init__(self, filter_json): self.filter_json = filter_json + def timeline_limit(self): + return self.filter_json.get("room", {}).get("timeline", {}).get(limit, 10) + def filter_public_user_data(self, events): return self._filter_on_key(events, ["public_user_data"]) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9914ff6f9c..2a0e045430 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -28,21 +28,26 @@ logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", - "limit", - "gap", - "sort", - "backfill", "filter", ]) +class TimelineBatch(collections.namedtuple("TimelineBatch", [ + "prev_batch", + "events", + "limited", +])): + __slots__ = [] + + def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if room needs to be part of the sync result. + """ + return bool(self.events) class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "room_id", - "limited", - "published", - "events", + "timeline", "state", - "prev_batch", "ephemeral", ])): __slots__ = [] @@ -51,13 +56,12 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ """Make the result appear empty if there are no updates. This is used to tell if room needs to be part of the sync result. """ - return bool(self.events or self.state or self.ephemeral) + return bool(self.timeline or self.state or self.ephemeral) class SyncResult(collections.namedtuple("SyncResult", [ "next_batch", # Token for the next sync - "private_user_data", # List of private events for the user. - "public_user_data", # List of public events for all users. + "presence", # List of presence events for the user. "rooms", # RoomSyncResult for each room. ])): __slots__ = [] @@ -133,12 +137,6 @@ class SyncHandler(BaseHandler): Returns: A Deferred SyncResult. """ - if sync_config.sort == "timeline,desc": - # TODO(mjark): Handle going through events in reverse order?. - # What does "most recent events" mean when applying the limits mean - # in this case? - raise NotImplementedError() - now_token = yield self.event_sources.get_current_token() presence_stream = self.event_sources.sources["presence"] @@ -155,20 +153,15 @@ class SyncHandler(BaseHandler): membership_list=[Membership.INVITE, Membership.JOIN] ) - # TODO (mjark): Does public mean "published"? - published_rooms = yield self.store.get_rooms(is_public=True) - published_room_ids = set(r["room_id"] for r in published_rooms) - rooms = [] for event in room_list: room_sync = yield self.initial_sync_for_room( - event.room_id, sync_config, now_token, published_room_ids + event.room_id, sync_config, now_token, ) rooms.append(room_sync) defer.returnValue(SyncResult( - public_user_data=presence, - private_user_data=[], + presence=presence, rooms=rooms, next_batch=now_token, )) @@ -192,7 +185,6 @@ class SyncHandler(BaseHandler): defer.returnValue(RoomSyncResult( room_id=room_id, - published=room_id in published_room_ids, events=recents, prev_batch=prev_batch_token, state=current_state_events, @@ -219,7 +211,6 @@ class SyncHandler(BaseHandler): presence, presence_key = yield presence_source.get_new_events_for_user( user=sync_config.user, from_key=since_token.presence_key, - limit=sync_config.limit, ) now_token = now_token.copy_and_replace("presence_key", presence_key) @@ -227,7 +218,6 @@ class SyncHandler(BaseHandler): typing, typing_key = yield typing_source.get_new_events_for_user( user=sync_config.user, from_key=since_token.typing_key, - limit=sync_config.limit, ) now_token = now_token.copy_and_replace("typing_key", typing_key) @@ -252,16 +242,18 @@ class SyncHandler(BaseHandler): published_rooms = yield self.store.get_rooms(is_public=True) published_room_ids = set(r["room_id"] for r in published_rooms) + timeline_limit = sync_config.filter.timeline_limit() + room_events, _ = yield self.store.get_room_events_stream( sync_config.user.to_string(), from_key=since_token.room_key, to_key=now_token.room_key, room_id=None, - limit=sync_config.limit + 1, + limit=timeline_limit + 1, ) rooms = [] - if len(room_events) <= sync_config.limit: + if len(room_events) <= timeline_limit: # There is no gap in any of the rooms. Therefore we can just # partition the new events by room and return them. events_by_room_id = {} @@ -365,8 +357,9 @@ class SyncHandler(BaseHandler): max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key end_key = room_key + timeline_limit = sync_config.filter.timeline_limit() - while limited and len(recents) < sync_config.limit and max_repeat: + while limited and len(recents) < timeline_limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, @@ -393,7 +386,9 @@ class SyncHandler(BaseHandler): "room_key", room_key ) - defer.returnValue((recents, prev_batch_token, limited)) + defer.returnValue(TimelineBatch( + events=recents, prev_batch=prev_batch_token, limited=limited + )) @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, @@ -408,7 +403,7 @@ class SyncHandler(BaseHandler): # TODO(mjark): Check for redactions we might have missed. - recents, prev_batch_token, limited = yield self.load_filtered_recents( + batch = yield self.load_filtered_recents( room_id, sync_config, now_token, since_token, ) @@ -437,11 +432,8 @@ class SyncHandler(BaseHandler): room_sync = RoomSyncResult( room_id=room_id, - published=room_id in published_room_ids, - events=recents, - prev_batch=prev_batch_token, + timeline=batch, state=state_events_delta, - limited=limited, ephemeral=typing_by_room.get(room_id, []) ) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index cac28b47b6..ea6600b1d5 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -36,47 +36,35 @@ class SyncRestServlet(RestServlet): GET parameters:: timeout(int): How long to wait for new events in milliseconds. - limit(int): Maxiumum number of events per room to return. - gap(bool): Create gaps the message history if limit is exceeded to - ensure that the client has the most recent messages. Defaults to - "true". - sort(str,str): tuple of sort key (e.g. "timeline") and direction - (e.g. "asc", "desc"). Defaults to "timeline,asc". since(batch_token): Batch token when asking for incremental deltas. set_presence(str): What state the device presence should be set to. default is "online". - backfill(bool): Should the HS request message history from other - servers. This may take a long time making it unsuitable for clients - expecting a prompt response. Defaults to "true". filter(filter_id): A filter to apply to the events returned. - filter_*: Filter override parameters. Response JSON:: { - "next_batch": // batch token for the next /sync - "private_user_data": // private events for this user. - "public_user_data": // public events for all users including the - // public events for this user. - "rooms": [{ // List of rooms with updates. - "room_id": // Id of the room being updated - "limited": // Was the per-room event limit exceeded? - "published": // Is the room published by our HS? - "event_map": // Map of EventID -> event JSON. - "events": { // The recent events in the room if gap is "true" - // otherwise the next events in the room. - "batch": [] // list of EventIDs in the "event_map". - "prev_batch": // back token for getting previous events. - } - "state": [] // list of EventIDs updating the current state to - // be what it should be at the end of the batch. - "ephemeral": [] + "next_batch": // batch token for the next /sync + "presence": // presence data for the user. + "rooms": { + "roomlist": [{ // List of rooms with updates. + "room_id": // Id of the room being updated + "event_map": // Map of EventID -> event JSON. + "timeline": { // The recent events in the room if gap is "true" + "limited": // Was the per-room event limit exceeded? + // otherwise the next events in the room. + "batch": [] // list of EventIDs in the "event_map". + "prev_batch": // back token for getting previous events. + } + "state": [] // list of EventIDs updating the current state to + // be what it should be at the end of the batch. + "ephemeral": [] }] + } } """ PATTERN = client_v2_pattern("/sync$") - ALLOWED_SORT = set(["timeline,asc", "timeline,desc"]) - ALLOWED_PRESENCE = set(["online", "offline", "idle"]) + ALLOWED_PRESENCE = set(["online", "offline"]) def __init__(self, hs): super(SyncRestServlet, self).__init__() @@ -90,45 +78,29 @@ class SyncRestServlet(RestServlet): user, token_id = yield self.auth.get_user_by_req(request) timeout = parse_integer(request, "timeout", default=0) - limit = parse_integer(request, "limit", required=True) - gap = parse_boolean(request, "gap", default=True) - sort = parse_string( - request, "sort", default="timeline,asc", - allowed_values=self.ALLOWED_SORT - ) since = parse_string(request, "since") set_presence = parse_string( request, "set_presence", default="online", allowed_values=self.ALLOWED_PRESENCE ) - backfill = parse_boolean(request, "backfill", default=False) filter_id = parse_string(request, "filter", default=None) logger.info( - "/sync: user=%r, timeout=%r, limit=%r, gap=%r, sort=%r, since=%r," - " set_presence=%r, backfill=%r, filter_id=%r" % ( - user, timeout, limit, gap, sort, since, set_presence, - backfill, filter_id + "/sync: user=%r, timeout=%r, since=%r," + " set_presence=%r, filter_id=%r" % ( + user, timeout, since, set_presence, filter_id ) ) - # TODO(mjark): Load filter and apply overrides. try: filter = yield self.filtering.get_user_filter( user.localpart, filter_id ) except: filter = Filter({}) - # filter = filter.apply_overrides(http_request) - # if filter.matches(event): - # # stuff sync_config = SyncConfig( user=user, - gap=gap, - limit=limit, - sort=sort, - backfill=backfill, filter=filter, ) @@ -144,11 +116,8 @@ class SyncRestServlet(RestServlet): time_now = self.clock.time_msec() response_content = { - "public_user_data": self.encode_user_data( - sync_result.public_user_data, filter, time_now - ), - "private_user_data": self.encode_user_data( - sync_result.private_user_data, filter, time_now + "presence": self.encode_user_data( + sync_result.presence, filter, time_now ), "rooms": self.encode_rooms( sync_result.rooms, filter, time_now, token_id -- cgit 1.5.1 From 9c311dfce59a035a4174149c5b05b1aac0f776e1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Oct 2015 11:04:23 +0100 Subject: Also bundle in sender --- synapse/handlers/_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 3a232cbeae..c488ee0f6d 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -132,6 +132,7 @@ class BaseHandler(object): "type": e.type, "state_key": e.state_key, "content": e.content, + "sender": e.sender, } for k, e in context.current_state.items() if e.type in ( -- cgit 1.5.1 From 40017a9a114aa917d7cb3231da08465f7500ab41 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Oct 2015 11:22:56 +0100 Subject: Add 'trusted_private_chat' to room creation presets --- synapse/api/constants.py | 1 + synapse/handlers/room.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 3385664394..008ee64727 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -83,3 +83,4 @@ class RejectedReason(object): class RoomCreationPreset(object): PRIVATE_CHAT = "private_chat" PUBLIC_CHAT = "public_chat" + TRUSTED_PRIVATE_CHAT = "trusted_private_chat" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index ac636255c2..3364a5de14 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -41,6 +41,11 @@ class RoomCreationHandler(BaseHandler): "history_visibility": "shared", "original_invitees_have_ops": False, }, + RoomCreationPreset.TRUSTED_PRIVATE_CHAT: { + "join_rules": JoinRules.INVITE, + "history_visibility": "shared", + "original_invitees_have_ops": True, + }, RoomCreationPreset.PUBLIC_CHAT: { "join_rules": JoinRules.PUBLIC, "history_visibility": "shared", -- cgit 1.5.1 From 49ebd472fab93ce8e8841cbd24ac209d56f15341 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Oct 2015 13:11:49 +0100 Subject: Explicitly add Create event as auth event --- synapse/handlers/federation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3aff80bf59..3882ba79ed 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -162,7 +162,7 @@ class FederationHandler(BaseHandler): auth_ids = [e_id for e_id, _ in e.auth_events] auth = { (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids + if e.event_id in auth_ids or e.type == EventTypes.Create } event_infos.append({ "event": e, @@ -1221,7 +1221,7 @@ class FederationHandler(BaseHandler): auth_ids = [e_id for e_id, _ in e.auth_events] auth = { (e.type, e.state_key): e for e in remote_auth_chain - if e.event_id in auth_ids + if e.event_id in auth_ids or e.type == EventTypes.Create } e.internal_metadata.outlier = True @@ -1339,6 +1339,7 @@ class FederationHandler(BaseHandler): (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids + or event.type == EventTypes.Create } ev.internal_metadata.outlier = True -- cgit 1.5.1 From 8fc52bc56a0813d40741f45164caa3230d3e00ec Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 2 Oct 2015 17:13:51 -0500 Subject: Allow synapse's useragent to be customized This will allow me to write tests which verify which server made HTTP requests in a federation context. --- synapse/config/server.py | 1 + synapse/http/client.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 4d12d49857..50c4afdcfe 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -26,6 +26,7 @@ class ServerConfig(Config): self.soft_file_limit = config["soft_file_limit"] self.daemonize = config.get("daemonize") self.print_pidfile = config.get("print_pidfile") + self.user_agent_override = config.get("user_agent_override") self.use_frozen_dicts = config.get("use_frozen_dicts", True) self.listeners = config.get("listeners", []) diff --git a/synapse/http/client.py b/synapse/http/client.py index 0933388c04..6adf35c7bf 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -67,7 +67,9 @@ class SimpleHttpClient(object): connectTimeout=15, contextFactory=hs.get_http_client_context_factory() ) - self.version_string = hs.version_string + self.user_agent = hs.config.user_agent_override + if self.user_agent is None: + self.user_agent = hs.version_string def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach @@ -112,7 +114,7 @@ class SimpleHttpClient(object): uri.encode("ascii"), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -131,7 +133,8 @@ class SimpleHttpClient(object): "POST", uri.encode("ascii"), headers=Headers({ - "Content-Type": ["application/json"] + b"Content-Type": [b"application/json"], + b"User-Agent": [self.user_agent], }), bodyProducer=FileBodyProducer(StringIO(json_str)) ) @@ -165,7 +168,7 @@ class SimpleHttpClient(object): "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], }) ) @@ -206,7 +209,7 @@ class SimpleHttpClient(object): "PUT", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], "Content-Type": ["application/json"] }), bodyProducer=FileBodyProducer(StringIO(json_str)) @@ -241,7 +244,7 @@ class CaptchaServerHttpClient(SimpleHttpClient): bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], }) ) -- cgit 1.5.1 From 471555b3a815968c4d7e41a1b99390c6a7917a21 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 5 Oct 2015 16:39:22 +0100 Subject: Move the rooms out into a room_map mapping from room_id to room. --- synapse/api/filtering.py | 8 ++++++- synapse/handlers/sync.py | 27 ++++++++-------------- synapse/rest/client/v2_alpha/sync.py | 44 +++++++++++++++++++++++++----------- 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index c066ce89ef..2d5431ba60 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -136,7 +136,13 @@ class Filter(object): self.filter_json = filter_json def timeline_limit(self): - return self.filter_json.get("room", {}).get("timeline", {}).get(limit, 10) + return self.filter_json.get("room", {}).get("timeline", {}).get("limit", 10) + + def presence_limit(self): + return self.filter_json.get("presence", {}).get("limit", 10) + + def ephemeral_limit(self): + return self.filter_json.get("room", {}).get("ephemeral", {}).get("limit", 10) def filter_public_user_data(self, events): return self._filter_on_key(events, ["public_user_data"]) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 2a0e045430..9d488fa251 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -31,6 +31,7 @@ SyncConfig = collections.namedtuple("SyncConfig", [ "filter", ]) + class TimelineBatch(collections.namedtuple("TimelineBatch", [ "prev_batch", "events", @@ -44,6 +45,7 @@ class TimelineBatch(collections.namedtuple("TimelineBatch", [ """ return bool(self.events) + class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "room_id", "timeline", @@ -125,11 +127,7 @@ class SyncHandler(BaseHandler): if since_token is None: return self.initial_sync(sync_config) else: - if sync_config.gap: - return self.incremental_sync_with_gap(sync_config, since_token) - else: - # TODO(mjark): Handle gapless sync - raise NotImplementedError() + return self.incremental_sync_with_gap(sync_config, since_token) @defer.inlineCallbacks def initial_sync(self, sync_config): @@ -174,7 +172,7 @@ class SyncHandler(BaseHandler): A Deferred RoomSyncResult. """ - recents, prev_batch_token, limited = yield self.load_filtered_recents( + batch = yield self.load_filtered_recents( room_id, sync_config, now_token, ) @@ -185,10 +183,8 @@ class SyncHandler(BaseHandler): defer.returnValue(RoomSyncResult( room_id=room_id, - events=recents, - prev_batch=prev_batch_token, + timeline=batch, state=current_state_events, - limited=limited, ephemeral=[], )) @@ -199,18 +195,13 @@ class SyncHandler(BaseHandler): Returns: A Deferred SyncResult. """ - if sync_config.sort == "timeline,desc": - # TODO(mjark): Handle going through events in reverse order?. - # What does "most recent events" mean when applying the limits mean - # in this case? - raise NotImplementedError() - now_token = yield self.event_sources.get_current_token() presence_source = self.event_sources.sources["presence"] presence, presence_key = yield presence_source.get_new_events_for_user( user=sync_config.user, from_key=since_token.presence_key, + limit=sync_config.filter.presence_limit(), ) now_token = now_token.copy_and_replace("presence_key", presence_key) @@ -218,6 +209,7 @@ class SyncHandler(BaseHandler): typing, typing_key = yield typing_source.get_new_events_for_user( user=sync_config.user, from_key=since_token.typing_key, + limit=sync_config.filter.ephemeral_limit(), ) now_token = now_token.copy_and_replace("typing_key", typing_key) @@ -295,8 +287,7 @@ class SyncHandler(BaseHandler): rooms.append(room_sync) defer.returnValue(SyncResult( - public_user_data=presence, - private_user_data=[], + presence=presence, rooms=rooms, next_batch=now_token, )) @@ -407,7 +398,7 @@ class SyncHandler(BaseHandler): room_id, sync_config, now_token, since_token, ) - logging.debug("Recents %r", recents) + logging.debug("Recents %r", batch) # TODO(mjark): This seems racy since this isn't being passed a # token to indicate what point in the stream this is diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index ea6600b1d5..1f3824d924 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.http.servlet import ( - RestServlet, parse_string, parse_integer, parse_boolean + RestServlet, parse_string, parse_integer ) from synapse.handlers.sync import SyncConfig from synapse.types import StreamToken @@ -46,8 +46,14 @@ class SyncRestServlet(RestServlet): "next_batch": // batch token for the next /sync "presence": // presence data for the user. "rooms": { - "roomlist": [{ // List of rooms with updates. - "room_id": // Id of the room being updated + "default": { + "invited": [], // Ids of invited rooms being updated. + "joined": [], // Ids of joined rooms being updated. + "archived": [] // Ids of archived rooms being updated. + } + } + "room_map": { + "${room_id}": { // Id of the room being updated "event_map": // Map of EventID -> event JSON. "timeline": { // The recent events in the room if gap is "true" "limited": // Was the per-room event limit exceeded? @@ -58,7 +64,7 @@ class SyncRestServlet(RestServlet): "state": [] // list of EventIDs updating the current state to // be what it should be at the end of the batch. "ephemeral": [] - }] + } } } """ @@ -115,13 +121,16 @@ class SyncRestServlet(RestServlet): time_now = self.clock.time_msec() + room_map, rooms = self.encode_rooms( + sync_result.rooms, filter, time_now, token_id + ) + response_content = { "presence": self.encode_user_data( sync_result.presence, filter, time_now ), - "rooms": self.encode_rooms( - sync_result.rooms, filter, time_now, token_id - ), + "room_map": room_map, + "rooms": rooms, "next_batch": sync_result.next_batch.to_string(), } @@ -131,10 +140,21 @@ class SyncRestServlet(RestServlet): return events def encode_rooms(self, rooms, filter, time_now, token_id): - return [ - self.encode_room(room, filter, time_now, token_id) - for room in rooms - ] + room_map = {} + joined = [] + for room in rooms: + room_map[room.room_id] = self.encode_room( + room, filter, time_now, token_id + ) + joined.append(room.room_id) + + return room_map, { + "default": { + "joined": joined, + "invited": [], + "archived": [], + } + } @staticmethod def encode_room(room, filter, time_now, token_id): @@ -159,7 +179,6 @@ class SyncRestServlet(RestServlet): ) recent_event_ids.append(event.event_id) result = { - "room_id": room.room_id, "event_map": event_map, "events": { "batch": recent_event_ids, @@ -167,7 +186,6 @@ class SyncRestServlet(RestServlet): }, "state": state_event_ids, "limited": room.limited, - "published": room.published, "ephemeral": room.ephemeral, } return result -- cgit 1.5.1 From 34d26d36874ab0bc7fa4dab0db08bd37ba4936ae Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 5 Oct 2015 19:10:47 -0500 Subject: Revert "Merge pull request #283 from matrix-org/erikj/atomic_join_federation" This reverts commit 5879edbb097d19c2b5f5e064841909e67d6018fe, reversing changes made to b43930d4c9479973fb7225818e2e51bec022c4e6. --- synapse/handlers/federation.py | 207 +++++++++++++++-------------------------- 1 file changed, 76 insertions(+), 131 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b315ff15cc..d3d172b7b4 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -125,72 +125,60 @@ class FederationHandler(BaseHandler): ) if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") + current_state = state - try: - event_stream_id, max_stream_id = yield self._persist_auth_tree( - auth_chain, state, event - ) - except AuthError as e: - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event.event_id, - ) + event_ids = set() + if state: + event_ids |= {e.event_id for e in state} + if auth_chain: + event_ids |= {e.event_id for e in auth_chain} - else: - event_ids = set() - if state: - event_ids |= {e.event_id for e in state} - if auth_chain: - event_ids |= {e.event_id for e in auth_chain} - - seen_ids = set( - (yield self.store.have_events(event_ids)).keys() - ) + seen_ids = set( + (yield self.store.have_events(event_ids)).keys() + ) - if state and auth_chain is not None: - # If we have any state or auth_chain given to us by the replication - # layer, then we should handle them (if we haven't before.) + if state and auth_chain is not None: + # If we have any state or auth_chain given to us by the replication + # layer, then we should handle them (if we haven't before.) - event_infos = [] + event_infos = [] - for e in itertools.chain(auth_chain, state): - if e.event_id in seen_ids: - continue - e.internal_metadata.outlier = True - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - event_infos.append({ - "event": e, - "auth_events": auth, - }) - seen_ids.add(e.event_id) + for e in itertools.chain(auth_chain, state): + if e.event_id in seen_ids: + continue + e.internal_metadata.outlier = True + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + event_infos.append({ + "event": e, + "auth_events": auth, + }) + seen_ids.add(e.event_id) - yield self._handle_new_events( - origin, - event_infos, - outliers=True - ) + yield self._handle_new_events( + origin, + event_infos, + outliers=True + ) - try: - _, event_stream_id, max_stream_id = yield self._handle_new_event( - origin, - event, - state=state, - backfilled=backfilled, - current_state=current_state, - ) - except AuthError as e: - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event.event_id, - ) + try: + _, event_stream_id, max_stream_id = yield self._handle_new_event( + origin, + event, + state=state, + backfilled=backfilled, + current_state=current_state, + ) + except AuthError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) # if we're receiving valid events from an origin, # it's probably a good idea to mark it as not in retry-state @@ -662,8 +650,35 @@ class FederationHandler(BaseHandler): # FIXME pass - event_stream_id, max_stream_id = yield self._persist_auth_tree( - auth_chain, state, event + ev_infos = [] + for e in itertools.chain(state, auth_chain): + if e.event_id == event.event_id: + continue + + e.internal_metadata.outlier = True + auth_ids = [e_id for e_id, _ in e.auth_events] + ev_infos.append({ + "event": e, + "auth_events": { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + }) + + yield self._handle_new_events(origin, ev_infos, outliers=True) + + auth_ids = [e_id for e_id, _ in event.auth_events] + auth_events = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + + _, event_stream_id, max_stream_id = yield self._handle_new_event( + origin, + new_event, + state=state, + current_state=state, + auth_events=auth_events, ) with PreserveLoggingContext(): @@ -1019,76 +1034,6 @@ class FederationHandler(BaseHandler): is_new_state=(not outliers and not backfilled), ) - @defer.inlineCallbacks - def _persist_auth_tree(self, auth_events, state, event): - """Checks the auth chain is valid (and passes auth checks) for the - state and event. Then persists the auth chain and state atomically. - Persists the event seperately. - - Returns: - 2-tuple of (event_stream_id, max_stream_id) from the persist_event - call for `event` - """ - events_to_context = {} - for e in itertools.chain(auth_events, state): - ctx = yield self.state_handler.compute_event_context( - e, outlier=True, - ) - events_to_context[e.event_id] = ctx - e.internal_metadata.outlier = True - - event_map = { - e.event_id: e - for e in auth_events - } - - create_event = None - for e in auth_events: - if (e.type, e.state_key) == (EventTypes.Create, ""): - create_event = e - break - - for e in itertools.chain(auth_events, state, [event]): - auth_for_e = { - (event_map[e_id].type, event_map[e_id].state_key): event_map[e_id] - for e_id, _ in e.auth_events - } - if create_event: - auth_for_e[(EventTypes.Create, "")] = create_event - - try: - self.auth.check(e, auth_events=auth_for_e) - except AuthError as err: - logger.warn( - "Rejecting %s because %s", - e.event_id, err.msg - ) - - if e == event: - raise - events_to_context[e.event_id].rejected = RejectedReason.AUTH_ERROR - - yield self.store.persist_events( - [ - (e, events_to_context[e.event_id]) - for e in itertools.chain(auth_events, state) - ], - is_new_state=False, - ) - - new_event_context = yield self.state_handler.compute_event_context( - event, old_state=state, outlier=False, - ) - - event_stream_id, max_stream_id = yield self.store.persist_event( - event, new_event_context, - backfilled=False, - is_new_state=True, - current_state=state, - ) - - defer.returnValue((event_stream_id, max_stream_id)) - @defer.inlineCallbacks def _prep_event(self, origin, event, state=None, backfilled=False, current_state=None, auth_events=None): -- cgit 1.5.1 From b28c7da0a4507825440fd801e1b6dbc5e6a454a7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 5 Oct 2015 20:49:39 -0500 Subject: Preserve version string in user agent --- synapse/config/server.py | 2 +- synapse/http/client.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 50c4afdcfe..5c2d6bfeab 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -26,7 +26,7 @@ class ServerConfig(Config): self.soft_file_limit = config["soft_file_limit"] self.daemonize = config.get("daemonize") self.print_pidfile = config.get("print_pidfile") - self.user_agent_override = config.get("user_agent_override") + self.user_agent_suffix = config.get("user_agent_suffix") self.use_frozen_dicts = config.get("use_frozen_dicts", True) self.listeners = config.get("listeners", []) diff --git a/synapse/http/client.py b/synapse/http/client.py index 6adf35c7bf..5017801773 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -67,9 +67,9 @@ class SimpleHttpClient(object): connectTimeout=15, contextFactory=hs.get_http_client_context_factory() ) - self.user_agent = hs.config.user_agent_override - if self.user_agent is None: - self.user_agent = hs.version_string + self.user_agent = hs.version_string + if hs.config.user_agent_suffix: + self.user_agent += " - " + hs.config.user_agent_suffix def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach -- cgit 1.5.1 From e0b466bcfdabcd70accdacbb72cfaf8616a9d18f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 6 Oct 2015 09:32:26 -0500 Subject: Use space not dash as delimiter --- synapse/http/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 5017801773..e0dda97ef6 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -69,7 +69,7 @@ class SimpleHttpClient(object): ) self.user_agent = hs.version_string if hs.config.user_agent_suffix: - self.user_agent += " - " + hs.config.user_agent_suffix + self.user_agent += " " + hs.config.user_agent_suffix def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach -- cgit 1.5.1 From 492beb62a823335fe6a6fb4c59a1bc246c3f57e2 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 6 Oct 2015 09:53:33 -0500 Subject: Use space not dash as delimiter --- synapse/http/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index e0dda97ef6..79c529291f 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -69,7 +69,7 @@ class SimpleHttpClient(object): ) self.user_agent = hs.version_string if hs.config.user_agent_suffix: - self.user_agent += " " + hs.config.user_agent_suffix + self.user_agent = "%s %s" % (self.user_agent, hs.config.user_agent_suffix,) def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach -- cgit 1.5.1 From d4bb28c59bfe6bca8cb2e5f7706fddca9a9aaf87 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 6 Oct 2015 09:58:21 -0500 Subject: Revert "Revert "Merge pull request #283 from matrix-org/erikj/atomic_join_federation"" This reverts commit 34d26d36874ab0bc7fa4dab0db08bd37ba4936ae. --- synapse/handlers/federation.py | 207 ++++++++++++++++++++++++++--------------- 1 file changed, 131 insertions(+), 76 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8d99101619..8197d8b2d0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -125,60 +125,72 @@ class FederationHandler(BaseHandler): ) if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") - current_state = state - event_ids = set() - if state: - event_ids |= {e.event_id for e in state} - if auth_chain: - event_ids |= {e.event_id for e in auth_chain} + try: + event_stream_id, max_stream_id = yield self._persist_auth_tree( + auth_chain, state, event + ) + except AuthError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) - seen_ids = set( - (yield self.store.have_events(event_ids)).keys() - ) + else: + event_ids = set() + if state: + event_ids |= {e.event_id for e in state} + if auth_chain: + event_ids |= {e.event_id for e in auth_chain} + + seen_ids = set( + (yield self.store.have_events(event_ids)).keys() + ) - if state and auth_chain is not None: - # If we have any state or auth_chain given to us by the replication - # layer, then we should handle them (if we haven't before.) + if state and auth_chain is not None: + # If we have any state or auth_chain given to us by the replication + # layer, then we should handle them (if we haven't before.) - event_infos = [] + event_infos = [] - for e in itertools.chain(auth_chain, state): - if e.event_id in seen_ids: - continue - e.internal_metadata.outlier = True - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - event_infos.append({ - "event": e, - "auth_events": auth, - }) - seen_ids.add(e.event_id) + for e in itertools.chain(auth_chain, state): + if e.event_id in seen_ids: + continue + e.internal_metadata.outlier = True + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + event_infos.append({ + "event": e, + "auth_events": auth, + }) + seen_ids.add(e.event_id) - yield self._handle_new_events( - origin, - event_infos, - outliers=True - ) + yield self._handle_new_events( + origin, + event_infos, + outliers=True + ) - try: - _, event_stream_id, max_stream_id = yield self._handle_new_event( - origin, - event, - state=state, - backfilled=backfilled, - current_state=current_state, - ) - except AuthError as e: - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event.event_id, - ) + try: + _, event_stream_id, max_stream_id = yield self._handle_new_event( + origin, + event, + state=state, + backfilled=backfilled, + current_state=current_state, + ) + except AuthError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) # if we're receiving valid events from an origin, # it's probably a good idea to mark it as not in retry-state @@ -650,35 +662,8 @@ class FederationHandler(BaseHandler): # FIXME pass - ev_infos = [] - for e in itertools.chain(state, auth_chain): - if e.event_id == event.event_id: - continue - - e.internal_metadata.outlier = True - auth_ids = [e_id for e_id, _ in e.auth_events] - ev_infos.append({ - "event": e, - "auth_events": { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - }) - - yield self._handle_new_events(origin, ev_infos, outliers=True) - - auth_ids = [e_id for e_id, _ in event.auth_events] - auth_events = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - - _, event_stream_id, max_stream_id = yield self._handle_new_event( - origin, - new_event, - state=state, - current_state=state, - auth_events=auth_events, + event_stream_id, max_stream_id = yield self._persist_auth_tree( + auth_chain, state, event ) with PreserveLoggingContext(): @@ -1034,6 +1019,76 @@ class FederationHandler(BaseHandler): is_new_state=(not outliers and not backfilled), ) + @defer.inlineCallbacks + def _persist_auth_tree(self, auth_events, state, event): + """Checks the auth chain is valid (and passes auth checks) for the + state and event. Then persists the auth chain and state atomically. + Persists the event seperately. + + Returns: + 2-tuple of (event_stream_id, max_stream_id) from the persist_event + call for `event` + """ + events_to_context = {} + for e in itertools.chain(auth_events, state): + ctx = yield self.state_handler.compute_event_context( + e, outlier=True, + ) + events_to_context[e.event_id] = ctx + e.internal_metadata.outlier = True + + event_map = { + e.event_id: e + for e in auth_events + } + + create_event = None + for e in auth_events: + if (e.type, e.state_key) == (EventTypes.Create, ""): + create_event = e + break + + for e in itertools.chain(auth_events, state, [event]): + auth_for_e = { + (event_map[e_id].type, event_map[e_id].state_key): event_map[e_id] + for e_id, _ in e.auth_events + } + if create_event: + auth_for_e[(EventTypes.Create, "")] = create_event + + try: + self.auth.check(e, auth_events=auth_for_e) + except AuthError as err: + logger.warn( + "Rejecting %s because %s", + e.event_id, err.msg + ) + + if e == event: + raise + events_to_context[e.event_id].rejected = RejectedReason.AUTH_ERROR + + yield self.store.persist_events( + [ + (e, events_to_context[e.event_id]) + for e in itertools.chain(auth_events, state) + ], + is_new_state=False, + ) + + new_event_context = yield self.state_handler.compute_event_context( + event, old_state=state, outlier=False, + ) + + event_stream_id, max_stream_id = yield self.store.persist_event( + event, new_event_context, + backfilled=False, + is_new_state=True, + current_state=state, + ) + + defer.returnValue((event_stream_id, max_stream_id)) + @defer.inlineCallbacks def _prep_event(self, origin, event, state=None, backfilled=False, current_state=None, auth_events=None): -- cgit 1.5.1 From 93cc60e80501c4d979c4c43dc5592e702ddfb358 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 6 Oct 2015 16:10:19 +0100 Subject: Remove log line that was generated whenever an error was created. We are now creating error objects that aren't raised so it's probably a bit too confusing to keep --- synapse/api/errors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index ee3045268f..d1356eb4d9 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -47,7 +47,6 @@ class CodeMessageException(RuntimeError): """An exception with integer code and message string attributes.""" def __init__(self, code, msg): - logger.info("%s: %s, %s", type(self).__name__, code, msg) super(CodeMessageException, self).__init__("%d: %s" % (code, msg)) self.code = code self.msg = msg -- cgit 1.5.1 From fcd9ba8802fcd111db08d2f51c8ac715c028cb1b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 6 Oct 2015 10:13:05 -0500 Subject: Fix lint errors --- synapse/federation/federation_client.py | 1 - synapse/rest/client/v1/room.py | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index bf22913d4f..06b0c7adcf 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -372,7 +372,6 @@ class FederationClient(FederationBase): pdu_dict = ret["event"] - logger.debug("Got response to make_join: %s", pdu_dict) defer.returnValue( diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index ba37061290..ff84affea3 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -17,7 +17,7 @@ from twisted.internet import defer from base import ClientV1RestServlet, client_path_pattern -from synapse.api.errors import SynapseError, Codes, AuthError +from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID, RoomAlias @@ -424,7 +424,6 @@ class RoomMembershipRestServlet(ClientV1RestServlet): ): raise SynapseError(400, "Missing user_id key.") - id_server = content["id_server"] medium = content["medium"] address = content["address"] @@ -556,7 +555,9 @@ class RoomMembershipRestServlet(ClientV1RestServlet): # TODO: Check for success token = data["token"] public_key = data["public_key"] - key_validity_url = "https://%s/_matrix/identity/api/v1/pubkey/isvalid" % (id_server,) + key_validity_url = "https://%s/_matrix/identity/api/v1/pubkey/isvalid" % ( + id_server, + ) defer.returnValue((token, public_key, key_validity_url)) @defer.inlineCallbacks -- cgit 1.5.1 From 1cacc71050a44f00ddf2226f907e9b794672671d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 6 Oct 2015 10:13:28 -0500 Subject: Add third party invites to auth_events for joins --- synapse/api/auth.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 6607d08488..adb9a776e0 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -677,6 +677,11 @@ class Auth(object): if e_type == Membership.JOIN: if member_event and not is_public: auth_ids.append(member_event.event_id) + if ThirdPartyInvites.has_join_keys(event.content): + key = (EventTypes.ThirdPartyInvite, event.content["token"]) + invite = current_state.get(key) + if invite: + auth_ids.append(invite.event_id) else: if member_event: auth_ids.append(member_event.event_id) -- cgit 1.5.1 From 61ee72517c96d1b25746e4baea0febcc63b405fd Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 6 Oct 2015 10:16:15 -0500 Subject: Remove merge thinko --- synapse/handlers/_base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 59c86187a9..faf99f5bd3 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -134,10 +134,6 @@ class BaseHandler(object): event ) - (event_stream_id, max_stream_id) = yield self.store.persist_event( - event, context=context - ) - federation_handler = self.hs.get_handlers().federation_handler if event.type == EventTypes.Member: -- cgit 1.5.1 From e3d3205cd953342ce84b8a148c4f469ce7b79b7a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 7 Oct 2015 15:55:20 +0100 Subject: Update the sync response to match the latest spec --- synapse/rest/client/v2_alpha/sync.py | 46 +++++++++++++++++------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 1f3824d924..84011918af 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -45,26 +45,29 @@ class SyncRestServlet(RestServlet): { "next_batch": // batch token for the next /sync "presence": // presence data for the user. - "rooms": { - "default": { "invited": [], // Ids of invited rooms being updated. "joined": [], // Ids of joined rooms being updated. "archived": [] // Ids of archived rooms being updated. } } - "room_map": { - "${room_id}": { // Id of the room being updated - "event_map": // Map of EventID -> event JSON. - "timeline": { // The recent events in the room if gap is "true" + "rooms": { + "joined": { // Joined rooms being updated. + "${room_id}": { // Id of the room being updated + "event_map": // Map of EventID -> event JSON. + "timeline": { // The recent events in the room if gap is "true" "limited": // Was the per-room event limit exceeded? - // otherwise the next events in the room. - "batch": [] // list of EventIDs in the "event_map". + // otherwise the next events in the room. + "events": [] // list of EventIDs in the "event_map". "prev_batch": // back token for getting previous events. + } + "state": {"events": []} // list of EventIDs updating the + // current state to be what it should + // be at the end of the batch. + "ephemeral": {"events": []} // list of event objects } - "state": [] // list of EventIDs updating the current state to - // be what it should be at the end of the batch. - "ephemeral": [] - } + }, + "invited": {}, // Ids of invited rooms being updated. + "archived": {} // Ids of archived rooms being updated. } } """ @@ -121,7 +124,7 @@ class SyncRestServlet(RestServlet): time_now = self.clock.time_msec() - room_map, rooms = self.encode_rooms( + rooms = self.encode_rooms( sync_result.rooms, filter, time_now, token_id ) @@ -129,7 +132,6 @@ class SyncRestServlet(RestServlet): "presence": self.encode_user_data( sync_result.presence, filter, time_now ), - "room_map": room_map, "rooms": rooms, "next_batch": sync_result.next_batch.to_string(), } @@ -140,20 +142,16 @@ class SyncRestServlet(RestServlet): return events def encode_rooms(self, rooms, filter, time_now, token_id): - room_map = {} - joined = [] + joined = {} for room in rooms: - room_map[room.room_id] = self.encode_room( + joined[room.room_id] = self.encode_room( room, filter, time_now, token_id ) - joined.append(room.room_id) - return room_map, { - "default": { - "joined": joined, - "invited": [], - "archived": [], - } + return { + "joined": joined, + "invited": {}, + "archived": {}, } @staticmethod -- cgit 1.5.1 From 91482cd6a0285faa837d7bd503855aa002cd3034 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 8 Oct 2015 11:22:15 +0100 Subject: Use raw string for regex here, otherwise \b is the backspace character. Fixes displayname matching. --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index f1952b5a0f..0e0c61dec8 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -186,7 +186,7 @@ class Pusher(object): if not display_name: return False return re.search( - "\b%s\b" % re.escape(display_name), ev['content']['body'], + r"\b%s\b" % re.escape(display_name), ev['content']['body'], flags=re.IGNORECASE ) is not None -- cgit 1.5.1 From dfef2b41aa3202b130661c3c423b2cf7d0dbba97 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 8 Oct 2015 15:17:43 +0100 Subject: Update the v2 room sync format to match the current v2 spec --- synapse/handlers/sync.py | 25 +++++++++++-------------- synapse/rest/client/v2_alpha/sync.py | 14 +++++++------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9d488fa251..76cca7c621 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -165,8 +165,7 @@ class SyncHandler(BaseHandler): )) @defer.inlineCallbacks - def initial_sync_for_room(self, room_id, sync_config, now_token, - published_room_ids): + def initial_sync_for_room(self, room_id, sync_config, now_token): """Sync a room for a client which is starting without any state Returns: A Deferred RoomSyncResult. @@ -230,10 +229,6 @@ class SyncHandler(BaseHandler): sync_config.user ) - # TODO (mjark): Does public mean "published"? - published_rooms = yield self.store.get_rooms(is_public=True) - published_room_ids = set(r["room_id"] for r in published_rooms) - timeline_limit = sync_config.filter.timeline_limit() room_events, _ = yield self.store.get_room_events_stream( @@ -268,11 +263,12 @@ class SyncHandler(BaseHandler): room_sync = RoomSyncResult( room_id=room_id, - published=room_id in published_room_ids, - events=recents, - prev_batch=prev_batch, + timeline=TimelineBatch( + events=recents, + prev_batch=prev_batch, + limited=False, + ), state=state, - limited=False, ephemeral=typing_by_room.get(room_id, []) ) if room_sync: @@ -344,11 +340,11 @@ class SyncHandler(BaseHandler): limited = True recents = [] filtering_factor = 2 - load_limit = max(sync_config.limit * filtering_factor, 100) + timeline_limit = sync_config.filter.timeline_limit() + load_limit = max(timeline_limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key end_key = room_key - timeline_limit = sync_config.filter.timeline_limit() while limited and len(recents) < timeline_limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( @@ -369,8 +365,9 @@ class SyncHandler(BaseHandler): limited = False max_repeat -= 1 - if len(recents) > sync_config.limit: - recents = recents[-sync_config.limit:] + if len(recents) > timeline_limit: + limited = True + recents = recents[-timeline_limit:] room_key = recents[0].internal_metadata.before prev_batch_token = now_token.copy_and_replace( diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 84011918af..97bf95acfb 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -158,7 +158,7 @@ class SyncRestServlet(RestServlet): def encode_room(room, filter, time_now, token_id): event_map = {} state_events = filter.filter_room_state(room.state) - recent_events = filter.filter_room_events(room.events) + recent_events = filter.filter_room_events(room.timeline.events) state_event_ids = [] recent_event_ids = [] for event in state_events: @@ -178,13 +178,13 @@ class SyncRestServlet(RestServlet): recent_event_ids.append(event.event_id) result = { "event_map": event_map, - "events": { - "batch": recent_event_ids, - "prev_batch": room.prev_batch.to_string(), + "timeline": { + "events": recent_event_ids, + "prev_batch": room.timeline.prev_batch.to_string(), + "limited": room.timeline.limited, }, - "state": state_event_ids, - "limited": room.limited, - "ephemeral": room.ephemeral, + "state": {"events": state_event_ids}, + "ephemeral": {"events": room.ephemeral}, } return result -- cgit 1.5.1 From dc720217489e2a8cf528255502fe448a85e1ff52 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 8 Oct 2015 17:19:42 +0100 Subject: Add a flag to initial sync to indicate we want rooms that the user has left --- synapse/handlers/message.py | 13 ++++++++----- synapse/rest/client/v1/initial_sync.py | 4 +++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 30949ff7a6..b70258697b 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -324,7 +324,8 @@ class MessageHandler(BaseHandler): ) @defer.inlineCallbacks - def snapshot_all_rooms(self, user_id=None, pagin_config=None, as_client_event=True): + def snapshot_all_rooms(self, user_id=None, pagin_config=None, + as_client_event=True, include_archived=False): """Retrieve a snapshot of all rooms the user is invited or has joined. This snapshot may include messages for all rooms where the user is @@ -335,17 +336,19 @@ class MessageHandler(BaseHandler): pagin_config (synapse.api.streams.PaginationConfig): The pagination config used to determine how many messages *PER ROOM* to return. as_client_event (bool): True to get events in client-server format. + include_archived (bool): True to get rooms that the user has left Returns: A list of dicts with "room_id" and "membership" keys for all rooms the user is currently invited or joined in on. Rooms where the user is joined on, may return a "messages" key with messages, depending on the specified PaginationConfig. """ + memberships = [Membership.INVITE, Membership.JOIN] + if include_archived: + memberships.append(Membership.LEAVE) + room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=user_id, - membership_list=[ - Membership.INVITE, Membership.JOIN, Membership.LEAVE - ] + user_id=user_id, membership_list=memberships ) user = UserID.from_string(user_id) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index bac68cc29f..52b7951b8f 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -29,10 +29,12 @@ class InitialSyncRestServlet(ClientV1RestServlet): as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) handler = self.handlers.message_handler + include_archived = request.args.get("archived", None) == "1" content = yield handler.snapshot_all_rooms( user_id=user.to_string(), pagin_config=pagination_config, - as_client_event=as_client_event + as_client_event=as_client_event, + include_archived=include_archived, ) defer.returnValue((200, content)) -- cgit 1.5.1 From 51ef7256472106aaca99de836ae73564ee78349c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 8 Oct 2015 18:13:02 +0100 Subject: Use 'true' rather than '1' for archived flag --- synapse/rest/client/v1/initial_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 52b7951b8f..52c7943400 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -29,7 +29,7 @@ class InitialSyncRestServlet(ClientV1RestServlet): as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) handler = self.handlers.message_handler - include_archived = request.args.get("archived", None) == "1" + include_archived = request.args.get("archived", None) == ["true"] content = yield handler.snapshot_all_rooms( user_id=user.to_string(), pagin_config=pagination_config, -- cgit 1.5.1 From c85c9125627a62c73711786723be12be30d7a81e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 9 Oct 2015 15:48:31 +0100 Subject: Add basic full text search impl. --- synapse/api/constants.py | 19 +++++++ synapse/handlers/__init__.py | 2 + synapse/handlers/search.py | 95 ++++++++++++++++++++++++++++++++++ synapse/rest/client/v1/room.py | 17 ++++++ synapse/storage/__init__.py | 2 + synapse/storage/_base.py | 2 +- synapse/storage/schema/delta/24/fts.py | 57 ++++++++++++++++++++ synapse/storage/search.py | 75 +++++++++++++++++++++++++++ 8 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 synapse/handlers/search.py create mode 100644 synapse/storage/schema/delta/24/fts.py create mode 100644 synapse/storage/search.py diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 008ee64727..7c7f9ff957 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -84,3 +84,22 @@ class RoomCreationPreset(object): PRIVATE_CHAT = "private_chat" PUBLIC_CHAT = "public_chat" TRUSTED_PRIVATE_CHAT = "trusted_private_chat" + + +class SearchConstraintTypes(object): + FTS = "fts" + EXACT = "exact" + PREFIX = "prefix" + SUBSTRING = "substring" + RANGE = "range" + + +class KnownRoomEventKeys(object): + CONTENT_BODY = "content.body" + CONTENT_MSGTYPE = "content.msgtype" + CONTENT_NAME = "content.name" + CONTENT_TOPIC = "content.topic" + + SENDER = "sender" + ORIGIN_SERVER_TS = "origin_server_ts" + ROOM_ID = "room_id" diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index 8725c3c420..87b4d381c7 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -32,6 +32,7 @@ from .sync import SyncHandler from .auth import AuthHandler from .identity import IdentityHandler from .receipts import ReceiptsHandler +from .search import SearchHandler class Handlers(object): @@ -68,3 +69,4 @@ class Handlers(object): self.sync_handler = SyncHandler(hs) self.auth_handler = AuthHandler(hs) self.identity_handler = IdentityHandler(hs) + self.search_handler = SearchHandler(hs) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py new file mode 100644 index 0000000000..8b997fc394 --- /dev/null +++ b/synapse/handlers/search.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import BaseHandler + +from synapse.api.constants import KnownRoomEventKeys, SearchConstraintTypes +from synapse.api.errors import SynapseError +from synapse.events.utils import serialize_event + +import logging + + +logger = logging.getLogger(__name__) + + +KEYS_TO_ALLOWED_CONSTRAINT_TYPES = { + KnownRoomEventKeys.CONTENT_BODY: [SearchConstraintTypes.FTS], + KnownRoomEventKeys.CONTENT_MSGTYPE: [SearchConstraintTypes.EXACT], + KnownRoomEventKeys.CONTENT_NAME: [SearchConstraintTypes.FTS, SearchConstraintTypes.EXACT, SearchConstraintTypes.SUBSTRING], + KnownRoomEventKeys.CONTENT_TOPIC: [SearchConstraintTypes.FTS], + KnownRoomEventKeys.SENDER: [SearchConstraintTypes.EXACT], + KnownRoomEventKeys.ORIGIN_SERVER_TS: [SearchConstraintTypes.RANGE], + KnownRoomEventKeys.ROOM_ID: [SearchConstraintTypes.EXACT], +} + + +class RoomConstraint(object): + def __init__(self, search_type, keys, value): + self.search_type = search_type + self.keys = keys + self.value = value + + @classmethod + def from_dict(cls, d): + search_type = d["type"] + keys = d["keys"] + + for key in keys: + if key not in KEYS_TO_ALLOWED_CONSTRAINT_TYPES: + raise SynapseError(400, "Unrecognized key %r", key) + + if search_type not in KEYS_TO_ALLOWED_CONSTRAINT_TYPES[key]: + raise SynapseError(400, "Disallowed constraint type %r for key %r", search_type, key) + + return cls(search_type, keys, d["value"]) + + +class SearchHandler(BaseHandler): + + def __init__(self, hs): + super(SearchHandler, self).__init__(hs) + + @defer.inlineCallbacks + def search(self, content): + constraint_dicts = content["search_categories"]["room_events"]["constraints"] + constraints = [RoomConstraint.from_dict(c)for c in constraint_dicts] + + fts = False + for c in constraints: + if c.search_type == SearchConstraintTypes.FTS: + if fts: + raise SynapseError(400, "Only one constraint can be FTS") + fts = True + + res = yield self.hs.get_datastore().search_msgs(constraints) + + time_now = self.hs.get_clock().time_msec() + + results = [ + { + "rank": r["rank"], + "result": serialize_event(r["result"], time_now) + } + for r in res + ] + + logger.info("returning: %r", results) + + results.sort(key=lambda r: -r["rank"]) + + defer.returnValue(results) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 23871f161e..35bd702a43 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -529,6 +529,22 @@ class RoomTypingRestServlet(ClientV1RestServlet): defer.returnValue((200, {})) +class SearchRestServlet(ClientV1RestServlet): + PATTERN = client_path_pattern( + "/search$" + ) + + @defer.inlineCallbacks + def on_POST(self, request): + auth_user, _ = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + results = yield self.handlers.search_handler.search(content) + + defer.returnValue((200, results)) + + def _parse_json(request): try: content = json.loads(request.content.read()) @@ -585,3 +601,4 @@ def register_servlets(hs, http_server): RoomInitialSyncRestServlet(hs).register(http_server) RoomRedactEventRestServlet(hs).register(http_server) RoomTypingRestServlet(hs).register(http_server) + SearchRestServlet(hs).register(http_server) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 340e59afcb..5f91ef77c0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -40,6 +40,7 @@ from .filtering import FilteringStore from .end_to_end_keys import EndToEndKeyStore from .receipts import ReceiptsStore +from .search import SearchStore import fnmatch @@ -79,6 +80,7 @@ class DataStore(RoomMemberStore, RoomStore, EventsStore, ReceiptsStore, EndToEndKeyStore, + SearchStore, ): def __init__(self, hs): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 693784ad38..218e708054 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -519,7 +519,7 @@ class SQLBaseStore(object): allow_none=False, desc="_simple_select_one_onecol"): """Executes a SELECT query on the named table, which is expected to - return a single row, returning a single column from it." + return a single row, returning a single column from it. Args: table : string giving the table name diff --git a/synapse/storage/schema/delta/24/fts.py b/synapse/storage/schema/delta/24/fts.py new file mode 100644 index 0000000000..5680332758 --- /dev/null +++ b/synapse/storage/schema/delta/24/fts.py @@ -0,0 +1,57 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from synapse.storage import get_statements +from synapse.storage.engines import PostgresEngine + +logger = logging.getLogger(__name__) + + +POSTGRES_SQL = """ +CREATE TABLE event_search ( + event_id TEXT, + room_id TEXT, + key TEXT, + vector tsvector +); + +INSERT INTO event_search SELECT + event_id, room_id, 'content.body', + to_tsvector('english', json::json->'content'->>'body') + FROM events NATURAL JOIN event_json WHERE type = 'm.room.message'; + +INSERT INTO event_search SELECT + event_id, room_id, 'content.name', + to_tsvector('english', json::json->'content'->>'name') + FROM events NATURAL JOIN event_json WHERE type = 'm.room.name'; + +INSERT INTO event_search SELECT + event_id, room_id, 'content.topic', + to_tsvector('english', json::json->'content'->>'topic') + FROM events NATURAL JOIN event_json WHERE type = 'm.room.topic'; + + +CREATE INDEX event_search_idx ON event_search USING gin(vector); +""" + + +def run_upgrade(cur, database_engine, *args, **kwargs): + if not isinstance(database_engine, PostgresEngine): + # We only support FTS for postgres currently. + return + + for statement in get_statements(POSTGRES_SQL.splitlines()): + cur.execute(statement) diff --git a/synapse/storage/search.py b/synapse/storage/search.py new file mode 100644 index 0000000000..eea4477765 --- /dev/null +++ b/synapse/storage/search.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from _base import SQLBaseStore +from synapse.api.constants import KnownRoomEventKeys, SearchConstraintTypes + + +class SearchStore(SQLBaseStore): + @defer.inlineCallbacks + def search_msgs(self, constraints): + clauses = [] + args = [] + fts = None + + for c in constraints: + local_clauses = [] + if c.search_type == SearchConstraintTypes.FTS: + fts = c.value + for key in c.keys: + local_clauses.append("key = ?") + args.append(key) + elif c.search_type == SearchConstraintTypes.EXACT: + for key in c.keys: + if key == KnownRoomEventKeys.ROOM_ID: + for value in c.value: + local_clauses.append("room_id = ?") + args.append(value) + clauses.append( + "(%s)" % (" OR ".join(local_clauses),) + ) + + sql = ( + "SELECT ts_rank_cd(vector, query) AS rank, event_id" + " FROM plainto_tsquery('english', ?) as query, event_search" + " WHERE vector @@ query" + ) + + for clause in clauses: + sql += " AND " + clause + + sql += " ORDER BY rank DESC" + + results = yield self._execute( + "search_msgs", self.cursor_to_dict, sql, *([fts] + args) + ) + + events = yield self._get_events([r["event_id"] for r in results]) + + event_map = { + ev.event_id: ev + for ev in events + } + + defer.returnValue([ + { + "rank": r["rank"], + "result": event_map[r["event_id"]] + } + for r in results + if r["event_id"] in event_map + ]) -- cgit 1.5.1 From c15cf6ac069386df3095b5c69af96f0c76ce5276 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 9 Oct 2015 18:50:15 +0100 Subject: Format the presence events correctly for v2 --- synapse/rest/client/v2_alpha/sync.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 97bf95acfb..f20b830eda 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -26,6 +26,7 @@ from synapse.events.utils import ( from synapse.api.filtering import Filter from ._base import client_v2_pattern +import copy import logging logger = logging.getLogger(__name__) @@ -129,7 +130,7 @@ class SyncRestServlet(RestServlet): ) response_content = { - "presence": self.encode_user_data( + "presence": self.encode_presence( sync_result.presence, filter, time_now ), "rooms": rooms, @@ -138,8 +139,13 @@ class SyncRestServlet(RestServlet): defer.returnValue((200, response_content)) - def encode_user_data(self, events, filter, time_now): - return events + def encode_presence(self, events, filter, time_now): + formatted = [] + for event in events: + event = copy.deepcopy(event) + event['sender'] = event['content'].pop('user_id'); + formatted.append(event) + return {"events": formatted} def encode_rooms(self, rooms, filter, time_now, token_id): joined = {} -- cgit 1.5.1 From 1b9802a0d99aafddd41088c94dc46bf88399e879 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 9 Oct 2015 19:13:08 +0100 Subject: Split the sections of EventStreamHandler.get_stream that handle presence into separate functions. This makes the code a bit easier to read, and means that we can reuse the logic when implementing the v2 sync API. --- synapse/handlers/events.py | 87 +++++++++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 891502c04f..92afa35d57 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -46,6 +46,56 @@ class EventStreamHandler(BaseHandler): self.notifier = hs.get_notifier() + @defer.inlineCallbacks + def started_stream(self, user): + """Tells the presence handler that we have started an eventstream for + the user: + + Args: + user (User): The user who started a stream. + Returns: + A deferred that completes once their presence has been updated. + """ + if user not in self._streams_per_user: + self._streams_per_user[user] = 0 + if user in self._stop_timer_per_user: + try: + self.clock.cancel_call_later( + self._stop_timer_per_user.pop(user) + ) + except: + logger.exception("Failed to cancel event timer") + else: + yield self.distributor.fire("started_user_eventstream", user) + + self._streams_per_user[user] += 1 + + def stopped_stream(self, user): + """If there are no streams for a user this starts a timer that will + notify the presence handler that we haven't got an event stream for + the user unless the user starts a new stream in 30 seconds. + + Args: + user (User): The user who stopped a stream. + """ + self._streams_per_user[user] -= 1 + if not self._streams_per_user[user]: + del self._streams_per_user[user] + + # 30 seconds of grace to allow the client to reconnect again + # before we think they're gone + def _later(): + logger.debug("_later stopped_user_eventstream %s", user) + + self._stop_timer_per_user.pop(user, None) + + return self.distributor.fire("stopped_user_eventstream", user) + + logger.debug("Scheduling _later: for %s", user) + self._stop_timer_per_user[user] = ( + self.clock.call_later(30, _later) + ) + @defer.inlineCallbacks @log_function def get_stream(self, auth_user_id, pagin_config, timeout=0, @@ -59,20 +109,7 @@ class EventStreamHandler(BaseHandler): try: if affect_presence: - if auth_user not in self._streams_per_user: - self._streams_per_user[auth_user] = 0 - if auth_user in self._stop_timer_per_user: - try: - self.clock.cancel_call_later( - self._stop_timer_per_user.pop(auth_user) - ) - except: - logger.exception("Failed to cancel event timer") - else: - yield self.distributor.fire( - "started_user_eventstream", auth_user - ) - self._streams_per_user[auth_user] += 1 + yield self.started_stream(auth_user) rm_handler = self.hs.get_handlers().room_member_handler @@ -114,27 +151,7 @@ class EventStreamHandler(BaseHandler): finally: if affect_presence: - self._streams_per_user[auth_user] -= 1 - if not self._streams_per_user[auth_user]: - del self._streams_per_user[auth_user] - - # 10 seconds of grace to allow the client to reconnect again - # before we think they're gone - def _later(): - logger.debug( - "_later stopped_user_eventstream %s", auth_user - ) - - self._stop_timer_per_user.pop(auth_user, None) - - return self.distributor.fire( - "stopped_user_eventstream", auth_user - ) - - logger.debug("Scheduling _later: for %s", auth_user) - self._stop_timer_per_user[auth_user] = ( - self.clock.call_later(30, _later) - ) + self.stopped_stream(auth_user) class EventHandler(BaseHandler): -- cgit 1.5.1 From 0a96a9a02371bd36970db7dfdb2d4c6e98e0200e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 9 Oct 2015 19:57:50 +0100 Subject: Set the user as online if they start polling the v2 sync --- synapse/rest/client/v2_alpha/sync.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index f20b830eda..3348b46c14 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -79,6 +79,7 @@ class SyncRestServlet(RestServlet): def __init__(self, hs): super(SyncRestServlet, self).__init__() self.auth = hs.get_auth() + self.event_stream_handler = hs.get_handlers().event_stream_handler self.sync_handler = hs.get_handlers().sync_handler self.clock = hs.get_clock() self.filtering = hs.get_filtering() @@ -119,9 +120,16 @@ class SyncRestServlet(RestServlet): else: since_token = None - sync_result = yield self.sync_handler.wait_for_sync_for_user( - sync_config, since_token=since_token, timeout=timeout - ) + if set_presence == "online": + yield self.event_stream_handler.started_stream(user) + + try: + sync_result = yield self.sync_handler.wait_for_sync_for_user( + sync_config, since_token=since_token, timeout=timeout + ) + finally: + if set_presence == "online": + self.event_stream_handler.stopped_stream(user) time_now = self.clock.time_msec() -- cgit 1.5.1 From 987803781e3870e1e1a3652612d833779d6cd290 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 9 Oct 2015 19:59:50 +0100 Subject: Fix some races in the synapse presence handler caused by not yielding on deferreds --- synapse/handlers/presence.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index e91e81831e..ce60642127 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -378,7 +378,7 @@ class PresenceHandler(BaseHandler): # TODO(paul): perform a presence push as part of start/stop poll so # we don't have to do this all the time - self.changed_presencelike_data(target_user, state) + yield self.changed_presencelike_data(target_user, state) def bump_presence_active_time(self, user, now=None): if now is None: @@ -422,12 +422,12 @@ class PresenceHandler(BaseHandler): @log_function def started_user_eventstream(self, user): # TODO(paul): Use "last online" state - self.set_state(user, user, {"presence": PresenceState.ONLINE}) + return self.set_state(user, user, {"presence": PresenceState.ONLINE}) @log_function def stopped_user_eventstream(self, user): # TODO(paul): Save current state as "last online" state - self.set_state(user, user, {"presence": PresenceState.OFFLINE}) + return self.set_state(user, user, {"presence": PresenceState.OFFLINE}) @defer.inlineCallbacks def user_joined_room(self, user, room_id): @@ -1263,6 +1263,11 @@ class UserPresenceCache(object): self.state = {"presence": PresenceState.OFFLINE} self.serial = None + def __repr__(self): + return "UserPresenceCache(state=%r, serial=%r)" % ( + self.state, self.serial + ) + def update(self, state, serial): assert("mtime_age" not in state) -- cgit 1.5.1 From c33f5c1a2414632f21183f41ecd4aef00e46a437 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Wed, 7 Oct 2015 14:45:57 +0100 Subject: Provide ability to login using CAS --- synapse/config/cas.py | 39 +++++++++++++++++++++++++ synapse/config/homeserver.py | 3 +- synapse/handlers/auth.py | 31 ++++++++++++++++++++ synapse/rest/client/v1/login.py | 64 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 synapse/config/cas.py diff --git a/synapse/config/cas.py b/synapse/config/cas.py new file mode 100644 index 0000000000..81d034e8f0 --- /dev/null +++ b/synapse/config/cas.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class CasConfig(Config): + """Cas Configuration + + cas_server_url: URL of CAS server + """ + + def read_config(self, config): + cas_config = config.get("cas_config", None) + if cas_config: + self.cas_enabled = True + self.cas_server_url = cas_config["server_url"] + else: + self.cas_enabled = False + self.cas_server_url = None + + def default_config(self, config_dir_path, server_name, **kwargs): + return """ + # Enable CAS for registration and login. + #cas_config: + # server_url: "https://cas-server.com" + """ diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index d77f045406..3039f3c0bf 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -26,12 +26,13 @@ from .metrics import MetricsConfig from .appservice import AppServiceConfig from .key import KeyConfig from .saml2 import SAML2Config +from .cas import CasConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, MetricsConfig, - AppServiceConfig, KeyConfig, SAML2Config, ): + AppServiceConfig, KeyConfig, SAML2Config, CasConfig): pass diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 793b3fcd8b..0ad28c4948 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -295,6 +295,37 @@ class AuthHandler(BaseHandler): refresh_token = yield self.issue_refresh_token(user_id) defer.returnValue((user_id, access_token, refresh_token)) + @defer.inlineCallbacks + def login_with_cas_user_id(self, user_id): + """ + Authenticates the user with the given user ID, intended to have been captured from a CAS response + + Args: + user_id (str): User ID + Returns: + A tuple of: + The user's ID. + The access token for the user's session. + The refresh token for the user's session. + Raises: + StoreError if there was a problem storing the token. + LoginError if there was an authentication problem. + """ + user_id, ignored = yield self._find_user_id_and_pwd_hash(user_id) + + logger.info("Logging in user %s", user_id) + access_token = yield self.issue_access_token(user_id) + refresh_token = yield self.issue_refresh_token(user_id) + defer.returnValue((user_id, access_token, refresh_token)) + + @defer.inlineCallbacks + def does_user_exist(self, user_id): + try: + yield self._find_user_id_and_pwd_hash(user_id) + defer.returnValue(True) + except LoginError: + defer.returnValue(False) + @defer.inlineCallbacks def _find_user_id_and_pwd_hash(self, user_id): """Checks to see if a user with the given id exists. Will check case diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index e580f71964..56e5cf79fe 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError +from synapse.api.errors import SynapseError, LoginError, Codes from synapse.types import UserID from base import ClientV1RestServlet, client_path_pattern @@ -27,6 +27,9 @@ from saml2 import BINDING_HTTP_POST from saml2 import config from saml2.client import Saml2Client +import xml.etree.ElementTree as ET +import requests + logger = logging.getLogger(__name__) @@ -35,16 +38,23 @@ class LoginRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login$") PASS_TYPE = "m.login.password" SAML2_TYPE = "m.login.saml2" + CAS_TYPE = "m.login.cas" def __init__(self, hs): super(LoginRestServlet, self).__init__(hs) self.idp_redirect_url = hs.config.saml2_idp_redirect_url self.saml2_enabled = hs.config.saml2_enabled + self.cas_enabled = hs.config.cas_enabled + + self.cas_server_url = hs.config.cas_server_url + self.servername = hs.config.server_name def on_GET(self, request): flows = [{"type": LoginRestServlet.PASS_TYPE}] if self.saml2_enabled: flows.append({"type": LoginRestServlet.SAML2_TYPE}) + if self.cas_enabled: + flows.append({"type": LoginRestServlet.CAS_TYPE}) return (200, {"flows": flows}) def on_OPTIONS(self, request): @@ -67,6 +77,12 @@ class LoginRestServlet(ClientV1RestServlet): "uri": "%s%s" % (self.idp_redirect_url, relay_state) } defer.returnValue((200, result)) + elif self.cas_enabled and (login_submission["type"] == LoginRestServlet.CAS_TYPE): + url = "%s/proxyValidate" % (self.cas_server_url) + parameters = {"ticket": login_submission["ticket"], "service": login_submission["service"]} + response = requests.get(url, verify=False, params=parameters) + result = yield self.do_cas_login(response.text) + defer.returnValue(result) else: raise SynapseError(400, "Bad login type.") except KeyError: @@ -100,6 +116,41 @@ class LoginRestServlet(ClientV1RestServlet): defer.returnValue((200, result)) + @defer.inlineCallbacks + def do_cas_login(self, cas_response_body): + root = ET.fromstring(cas_response_body) + if not root.tag.endswith("serviceResponse"): + raise LoginError(401, "Invalid CAS response", errcode=Codes.UNAUTHORIZED) + if not root[0].tag.endswith("authenticationSuccess"): + raise LoginError(401, "Unsuccessful CAS response", errcode=Codes.UNAUTHORIZED) + for child in root[0]: + if child.tag.endswith("user"): + user = child.text + user_id = "@%s:%s" % (user, self.servername) + auth_handler = self.handlers.auth_handler + user_exists = yield auth_handler.does_user_exist(user_id) + if user_exists: + user_id, access_token, refresh_token = yield auth_handler.login_with_cas_user_id(user_id) + result = { + "user_id": user_id, # may have changed + "access_token": access_token, + "refresh_token": refresh_token, + "home_server": self.hs.hostname, + } + + else: + user_id, access_token = yield self.handlers.registration_handler.register(localpart=user) + result = { + "user_id": user_id, # may have changed + "access_token": access_token, + "home_server": self.hs.hostname, + } + + defer.returnValue((200, result)) + + + raise LoginError(401, "Invalid CAS response", errcode=Codes.UNAUTHORIZED) + class LoginFallbackRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login/fallback$") @@ -173,6 +224,15 @@ class SAML2RestServlet(ClientV1RestServlet): defer.returnValue(None) defer.returnValue((200, {"status": "not_authenticated"})) +class CasRestServlet(ClientV1RestServlet): + PATTERN = client_path_pattern("/login/cas") + + def __init__(self, hs): + super(CasRestServlet, self).__init__(hs) + self.cas_server_url = hs.config.cas_server_url + + def on_GET(self, request): + return (200, {"serverUrl": self.cas_server_url}) def _parse_json(request): try: @@ -188,4 +248,6 @@ def register_servlets(hs, http_server): LoginRestServlet(hs).register(http_server) if hs.config.saml2_enabled: SAML2RestServlet(hs).register(http_server) + if hs.config.cas_enabled: + CasRestServlet(hs).register(http_server) # TODO PasswordResetRestServlet(hs).register(http_server) -- cgit 1.5.1 From 22112f8d14d1fcdb567c75484b3717e931d705db Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Thu, 8 Oct 2015 23:34:04 +0100 Subject: Formatting changes --- synapse/handlers/auth.py | 3 ++- synapse/rest/client/v1/login.py | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 0ad28c4948..484f719253 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -298,7 +298,8 @@ class AuthHandler(BaseHandler): @defer.inlineCallbacks def login_with_cas_user_id(self, user_id): """ - Authenticates the user with the given user ID, intended to have been captured from a CAS response + Authenticates the user with the given user ID, + intended to have been captured from a CAS response Args: user_id (str): User ID diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 56e5cf79fe..4de5f19591 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -77,9 +77,13 @@ class LoginRestServlet(ClientV1RestServlet): "uri": "%s%s" % (self.idp_redirect_url, relay_state) } defer.returnValue((200, result)) - elif self.cas_enabled and (login_submission["type"] == LoginRestServlet.CAS_TYPE): + elif self.cas_enabled and (login_submission["type"] == + LoginRestServlet.CAS_TYPE): url = "%s/proxyValidate" % (self.cas_server_url) - parameters = {"ticket": login_submission["ticket"], "service": login_submission["service"]} + parameters = { + "ticket": login_submission["ticket"], + "service": login_submission["service"] + } response = requests.get(url, verify=False, params=parameters) result = yield self.do_cas_login(response.text) defer.returnValue(result) @@ -130,7 +134,8 @@ class LoginRestServlet(ClientV1RestServlet): auth_handler = self.handlers.auth_handler user_exists = yield auth_handler.does_user_exist(user_id) if user_exists: - user_id, access_token, refresh_token = yield auth_handler.login_with_cas_user_id(user_id) + user_id, access_token, refresh_token = yield + auth_handler.login_with_cas_user_id(user_id) result = { "user_id": user_id, # may have changed "access_token": access_token, @@ -139,7 +144,8 @@ class LoginRestServlet(ClientV1RestServlet): } else: - user_id, access_token = yield self.handlers.registration_handler.register(localpart=user) + user_id, access_token = yield + self.handlers.registration_handler.register(localpart=user) result = { "user_id": user_id, # may have changed "access_token": access_token, @@ -148,7 +154,6 @@ class LoginRestServlet(ClientV1RestServlet): defer.returnValue((200, result)) - raise LoginError(401, "Invalid CAS response", errcode=Codes.UNAUTHORIZED) @@ -224,6 +229,7 @@ class SAML2RestServlet(ClientV1RestServlet): defer.returnValue(None) defer.returnValue((200, {"status": "not_authenticated"})) + class CasRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login/cas") @@ -234,6 +240,7 @@ class CasRestServlet(ClientV1RestServlet): def on_GET(self, request): return (200, {"serverUrl": self.cas_server_url}) + def _parse_json(request): try: content = json.loads(request.content.read()) -- cgit 1.5.1 From 625e13bfde35a3c6fdd2b3e8263838ec4d4fbcc3 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Fri, 9 Oct 2015 11:02:56 +0100 Subject: Add get_raw method to SimpleHttpClient, use this in CAS auth rather than requests --- synapse/http/client.py | 55 +++++++++++++++++++++++++++-------------- synapse/rest/client/v1/login.py | 13 ++++++---- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 79c529291f..ca642a7a06 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -160,16 +160,40 @@ class SimpleHttpClient(object): On a non-2xx HTTP response. The response body will be used as the error message. """ + body = yield self.get_raw(uri, args) + defer.returnValue(json.loads(body)) + + @defer.inlineCallbacks + def put_json(self, uri, json_body, args={}): + """ Puts some json to the given URI. + + Args: + uri (str): The URI to request, not including query parameters + json_body (dict): The JSON to put in the HTTP body, + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + Returns: + Deferred: Succeeds when we get *any* 2xx HTTP response, with the + HTTP body as JSON. + Raises: + On a non-2xx HTTP response. + """ if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) + json_str = encode_canonical_json(json_body) + response = yield self.request( - "GET", + "PUT", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.user_agent], - }) + b"User-Agent": [self.version_string], + "Content-Type": ["application/json"] + }), + bodyProducer=FileBodyProducer(StringIO(json_str)) ) body = yield preserve_context_over_fn(readBody, response) @@ -183,46 +207,39 @@ class SimpleHttpClient(object): raise CodeMessageException(response.code, body) @defer.inlineCallbacks - def put_json(self, uri, json_body, args={}): - """ Puts some json to the given URI. + def get_raw(self, uri, args={}): + """ Gets raw text from the given URI. Args: uri (str): The URI to request, not including query parameters - json_body (dict): The JSON to put in the HTTP body, args (dict): A dictionary used to create query strings, defaults to None. **Note**: The value of each key is assumed to be an iterable and *not* a string. Returns: Deferred: Succeeds when we get *any* 2xx HTTP response, with the - HTTP body as JSON. + HTTP body at text. Raises: - On a non-2xx HTTP response. + On a non-2xx HTTP response. The response body will be used as the + error message. """ if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) - json_str = encode_canonical_json(json_body) - response = yield self.request( - "PUT", + "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.user_agent], - "Content-Type": ["application/json"] - }), - bodyProducer=FileBodyProducer(StringIO(json_str)) + b"User-Agent": [self.version_string], + }) ) body = yield preserve_context_over_fn(readBody, response) if 200 <= response.code < 300: - defer.returnValue(json.loads(body)) + defer.returnValue(body) else: - # NB: This is explicitly not json.loads(body)'d because the contract - # of CodeMessageException is a *string* message. Callers can always - # load it into JSON if they want. raise CodeMessageException(response.code, body) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 4de5f19591..f5cd6a1960 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -16,6 +16,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, LoginError, Codes +from synapse.http.client import SimpleHttpClient from synapse.types import UserID from base import ClientV1RestServlet, client_path_pattern @@ -28,7 +29,6 @@ from saml2 import config from saml2.client import Saml2Client import xml.etree.ElementTree as ET -import requests logger = logging.getLogger(__name__) @@ -79,13 +79,16 @@ class LoginRestServlet(ClientV1RestServlet): defer.returnValue((200, result)) elif self.cas_enabled and (login_submission["type"] == LoginRestServlet.CAS_TYPE): - url = "%s/proxyValidate" % (self.cas_server_url) - parameters = { + # TODO: get this from the homeserver rather than creating a new one for + # each request + http_client = SimpleHttpClient(self.hs) + uri = "%s/proxyValidate" % (self.cas_server_url,) + args = { "ticket": login_submission["ticket"], "service": login_submission["service"] } - response = requests.get(url, verify=False, params=parameters) - result = yield self.do_cas_login(response.text) + body = yield http_client.get_raw(uri, args) + result = yield self.do_cas_login(body) defer.returnValue(result) else: raise SynapseError(400, "Bad login type.") -- cgit 1.5.1 From e52f4dc5995fccd96a2a4084dc68a05da5a16838 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Fri, 9 Oct 2015 11:04:07 +0100 Subject: Use UserId to create FQ user id --- synapse/rest/client/v1/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index f5cd6a1960..05095e7d6e 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -133,7 +133,7 @@ class LoginRestServlet(ClientV1RestServlet): for child in root[0]: if child.tag.endswith("user"): user = child.text - user_id = "@%s:%s" % (user, self.servername) + user_id = UserID.create(user, self.hs.hostname).to_string() auth_handler = self.handlers.auth_handler user_exists = yield auth_handler.does_user_exist(user_id) if user_exists: -- cgit 1.5.1 From a9c299c0befc5cfc10ed1a5282b6002a43b9b462 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Fri, 9 Oct 2015 11:04:30 +0100 Subject: Fix my broken line splitting --- synapse/rest/client/v1/login.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 05095e7d6e..1bd93526ad 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -137,8 +137,9 @@ class LoginRestServlet(ClientV1RestServlet): auth_handler = self.handlers.auth_handler user_exists = yield auth_handler.does_user_exist(user_id) if user_exists: - user_id, access_token, refresh_token = yield - auth_handler.login_with_cas_user_id(user_id) + user_id, access_token, refresh_token = ( + yield auth_handler.login_with_cas_user_id(user_id) + ) result = { "user_id": user_id, # may have changed "access_token": access_token, @@ -147,8 +148,9 @@ class LoginRestServlet(ClientV1RestServlet): } else: - user_id, access_token = yield - self.handlers.registration_handler.register(localpart=user) + user_id, access_token = ( + yield self.handlers.registration_handler.register(localpart=user) + ) result = { "user_id": user_id, # may have changed "access_token": access_token, -- cgit 1.5.1 From 95f7661170c842966e14b0a274347e73b90f1134 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Fri, 9 Oct 2015 11:05:02 +0100 Subject: Raise LoginError if CasResponse doensn't contain user --- synapse/rest/client/v1/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 1bd93526ad..a99dcaab6f 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -157,7 +157,7 @@ class LoginRestServlet(ClientV1RestServlet): "home_server": self.hs.hostname, } - defer.returnValue((200, result)) + defer.returnValue((200, result)) raise LoginError(401, "Invalid CAS response", errcode=Codes.UNAUTHORIZED) -- cgit 1.5.1 From a80ef851f7e624b9eee91f134b233f3c0742bb3e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 10 Oct 2015 12:35:39 +0100 Subject: Fix previous merge to s/version_string/user_agent/ --- synapse/http/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index ca642a7a06..9a5869abee 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -190,7 +190,7 @@ class SimpleHttpClient(object): "PUT", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], "Content-Type": ["application/json"] }), bodyProducer=FileBodyProducer(StringIO(json_str)) @@ -231,7 +231,7 @@ class SimpleHttpClient(object): "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [self.version_string], + b"User-Agent": [self.user_agent], }) ) -- cgit 1.5.1 From 782f7fb4899ef078b422ce779e931e156fde15be Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 10 Oct 2015 18:24:44 +0100 Subject: add steve to authors --- AUTHORS.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 54ced67000..58a67c6b12 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -44,4 +44,7 @@ Eric Myhre repository API. Muthu Subramanian - * Add SAML2 support for registration and logins. + * Add SAML2 support for registration and login. + +Steven Hammerton + * Add CAS support for registration and login. -- cgit 1.5.1 From 61561b9df791ec90e287e535cc75831c2016bf36 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Oct 2015 10:49:53 +0100 Subject: Keep FTS indexes up to date. Only search through rooms currently joined --- synapse/handlers/search.py | 31 ++++++++++++++++++++++--------- synapse/rest/client/v1/room.py | 2 +- synapse/storage/events.py | 2 ++ synapse/storage/room.py | 22 ++++++++++++++++++++++ synapse/storage/schema/delta/24/fts.py | 3 ++- synapse/storage/search.py | 7 ++++++- 6 files changed, 55 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 8b997fc394..b6bdb752e9 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -65,7 +65,7 @@ class SearchHandler(BaseHandler): super(SearchHandler, self).__init__(hs) @defer.inlineCallbacks - def search(self, content): + def search(self, user, content): constraint_dicts = content["search_categories"]["room_events"]["constraints"] constraints = [RoomConstraint.from_dict(c)for c in constraint_dicts] @@ -76,20 +76,33 @@ class SearchHandler(BaseHandler): raise SynapseError(400, "Only one constraint can be FTS") fts = True - res = yield self.hs.get_datastore().search_msgs(constraints) + rooms = yield self.store.get_rooms_for_user( + user.to_string(), + ) - time_now = self.hs.get_clock().time_msec() + # For some reason the list of events contains duplicates + # TODO(paul): work out why because I really don't think it should + room_ids = set(r.room_id for r in rooms) - results = [ - { + res = yield self.store.search_msgs(room_ids, constraints) + + time_now = self.clock.time_msec() + + results = { + r["result"].event_id: { "rank": r["rank"], "result": serialize_event(r["result"], time_now) } for r in res - ] + } logger.info("returning: %r", results) - results.sort(key=lambda r: -r["rank"]) - - defer.returnValue(results) + defer.returnValue({ + "search_categories": { + "room_events": { + "results": results, + "count": len(results) + } + } + }) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 35bd702a43..94adabca62 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -540,7 +540,7 @@ class SearchRestServlet(ClientV1RestServlet): content = _parse_json(request) - results = yield self.handlers.search_handler.search(content) + results = yield self.handlers.search_handler.search(auth_user, content) defer.returnValue((200, results)) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 416ef6af93..e6c1abfc27 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -307,6 +307,8 @@ class EventsStore(SQLBaseStore): self._store_room_name_txn(txn, event) elif event.type == EventTypes.Topic: self._store_room_topic_txn(txn, event) + elif event.type == EventTypes.Message: + self._store_room_message_txn(txn, event) elif event.type == EventTypes.Redaction: self._store_redaction(txn, event) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 5e07b7e0e5..e4e830944a 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -175,6 +175,10 @@ class RoomStore(SQLBaseStore): }, ) + self._store_event_search_txn( + txn, event, "content.topic", event.content["topic"] + ) + def _store_room_name_txn(self, txn, event): if hasattr(event, "content") and "name" in event.content: self._simple_insert_txn( @@ -187,6 +191,24 @@ class RoomStore(SQLBaseStore): } ) + self._store_event_search_txn( + txn, event, "content.name", event.content["name"] + ) + + def _store_room_message_txn(self, txn, event): + if hasattr(event, "content") and "body" in event.content: + self._store_event_search_txn( + txn, event, "content.body", event.content["body"] + ) + + def _store_event_search_txn(self, txn, event, key, value): + sql = ( + "INSERT INTO event_search (event_id, room_id, key, vector)" + " VALUES (?,?,?,to_tsvector('english', ?))" + ) + + txn.execute(sql, (event.event_id, event.room_id, key, value,)) + @cachedInlineCallbacks() def get_room_name_and_aliases(self, room_id): def f(txn): diff --git a/synapse/storage/schema/delta/24/fts.py b/synapse/storage/schema/delta/24/fts.py index 5680332758..05f1605fdd 100644 --- a/synapse/storage/schema/delta/24/fts.py +++ b/synapse/storage/schema/delta/24/fts.py @@ -44,7 +44,8 @@ INSERT INTO event_search SELECT FROM events NATURAL JOIN event_json WHERE type = 'm.room.topic'; -CREATE INDEX event_search_idx ON event_search USING gin(vector); +CREATE INDEX event_search_fts_idx ON event_search USING gin(vector); +CREATE INDEX event_search_ev_idx ON event_search(event_id); """ diff --git a/synapse/storage/search.py b/synapse/storage/search.py index eea4477765..e66b5f9edc 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -21,11 +21,16 @@ from synapse.api.constants import KnownRoomEventKeys, SearchConstraintTypes class SearchStore(SQLBaseStore): @defer.inlineCallbacks - def search_msgs(self, constraints): + def search_msgs(self, room_ids, constraints): clauses = [] args = [] fts = None + clauses.append( + "room_id IN (%s)" % (",".join(["?"] * len(room_ids)),) + ) + args.extend(room_ids) + for c in constraints: local_clauses = [] if c.search_type == SearchConstraintTypes.FTS: -- cgit 1.5.1 From ae72e247fa478a541c837aaa7663aa3ca01ba840 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Oct 2015 10:50:46 +0100 Subject: PEP8 --- synapse/handlers/search.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index b6bdb752e9..9dc474aa56 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -30,7 +30,11 @@ logger = logging.getLogger(__name__) KEYS_TO_ALLOWED_CONSTRAINT_TYPES = { KnownRoomEventKeys.CONTENT_BODY: [SearchConstraintTypes.FTS], KnownRoomEventKeys.CONTENT_MSGTYPE: [SearchConstraintTypes.EXACT], - KnownRoomEventKeys.CONTENT_NAME: [SearchConstraintTypes.FTS, SearchConstraintTypes.EXACT, SearchConstraintTypes.SUBSTRING], + KnownRoomEventKeys.CONTENT_NAME: [ + SearchConstraintTypes.FTS, + SearchConstraintTypes.EXACT, + SearchConstraintTypes.SUBSTRING, + ], KnownRoomEventKeys.CONTENT_TOPIC: [SearchConstraintTypes.FTS], KnownRoomEventKeys.SENDER: [SearchConstraintTypes.EXACT], KnownRoomEventKeys.ORIGIN_SERVER_TS: [SearchConstraintTypes.RANGE], @@ -54,7 +58,10 @@ class RoomConstraint(object): raise SynapseError(400, "Unrecognized key %r", key) if search_type not in KEYS_TO_ALLOWED_CONSTRAINT_TYPES[key]: - raise SynapseError(400, "Disallowed constraint type %r for key %r", search_type, key) + raise SynapseError( + 400, + "Disallowed constraint type %r for key %r", search_type, key + ) return cls(search_type, keys, d["value"]) -- cgit 1.5.1 From 7845f62c2207e9fa51f7a0aa7b60b49cf6436696 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Mon, 12 Oct 2015 10:52:43 +0100 Subject: Parse both user and attributes from CAS response --- synapse/rest/client/v1/login.py | 64 ++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index a99dcaab6f..0e12880ab5 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -125,6 +125,34 @@ class LoginRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def do_cas_login(self, cas_response_body): + (user, attributes) = self.parse_cas_response(cas_response_body) + user_id = UserID.create(user, self.hs.hostname).to_string() + auth_handler = self.handlers.auth_handler + user_exists = yield auth_handler.does_user_exist(user_id) + if user_exists: + user_id, access_token, refresh_token = ( + yield auth_handler.login_with_cas_user_id(user_id) + ) + result = { + "user_id": user_id, # may have changed + "access_token": access_token, + "refresh_token": refresh_token, + "home_server": self.hs.hostname, + } + + else: + user_id, access_token = ( + yield self.handlers.registration_handler.register(localpart=user) + ) + result = { + "user_id": user_id, # may have changed + "access_token": access_token, + "home_server": self.hs.hostname, + } + + defer.returnValue((200, result)) + + def parse_cas_response(self, cas_response_body): root = ET.fromstring(cas_response_body) if not root.tag.endswith("serviceResponse"): raise LoginError(401, "Invalid CAS response", errcode=Codes.UNAUTHORIZED) @@ -133,33 +161,17 @@ class LoginRestServlet(ClientV1RestServlet): for child in root[0]: if child.tag.endswith("user"): user = child.text - user_id = UserID.create(user, self.hs.hostname).to_string() - auth_handler = self.handlers.auth_handler - user_exists = yield auth_handler.does_user_exist(user_id) - if user_exists: - user_id, access_token, refresh_token = ( - yield auth_handler.login_with_cas_user_id(user_id) - ) - result = { - "user_id": user_id, # may have changed - "access_token": access_token, - "refresh_token": refresh_token, - "home_server": self.hs.hostname, - } - - else: - user_id, access_token = ( - yield self.handlers.registration_handler.register(localpart=user) - ) - result = { - "user_id": user_id, # may have changed - "access_token": access_token, - "home_server": self.hs.hostname, - } - - defer.returnValue((200, result)) + if child.tag.endswith("attributes"): + attributes = {} + for attribute in child: + if "}" in attribute.tag: + attributes[attribute.tag.split("}")[1]] = attribute.text + else: + attributes[attribute.tag] = attribute.text + if user is None or attributes is None: + raise LoginError(401, "Invalid CAS response", errcode=Codes.UNAUTHORIZED) - raise LoginError(401, "Invalid CAS response", errcode=Codes.UNAUTHORIZED) + return (user, attributes) class LoginFallbackRestServlet(ClientV1RestServlet): -- cgit 1.5.1 From 76421c496d1ee4ba5ea97fb24466156d0ddc0723 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Mon, 12 Oct 2015 11:11:49 +0100 Subject: Allow optional config params for a required attribute and it's value, if specified any CAS user must have the given attribute and the value must equal --- synapse/config/cas.py | 15 +++++++++++++++ synapse/rest/client/v1/login.py | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/synapse/config/cas.py b/synapse/config/cas.py index 81d034e8f0..4d1dd8cc7b 100644 --- a/synapse/config/cas.py +++ b/synapse/config/cas.py @@ -27,13 +27,28 @@ class CasConfig(Config): if cas_config: self.cas_enabled = True self.cas_server_url = cas_config["server_url"] + + if "required_attribute" in cas_config: + self.cas_required_attribute = cas_config["required_attribute"] + else: + self.cas_required_attribute = None + + if "required_attribute_value" in cas_config: + self.cas_required_attribute_value = cas_config["required_attribute_value"] + else: + self.cas_required_attribute_value = None + else: self.cas_enabled = False self.cas_server_url = None + self.cas_required_attribute = None + self.cas_required_attribute_value = None def default_config(self, config_dir_path, server_name, **kwargs): return """ # Enable CAS for registration and login. #cas_config: # server_url: "https://cas-server.com" + # #required_attribute: something + # #required_attribute_value: true """ diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 0e12880ab5..1e62beaff8 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -45,8 +45,9 @@ class LoginRestServlet(ClientV1RestServlet): self.idp_redirect_url = hs.config.saml2_idp_redirect_url self.saml2_enabled = hs.config.saml2_enabled self.cas_enabled = hs.config.cas_enabled - self.cas_server_url = hs.config.cas_server_url + self.cas_required_attribute = hs.config.cas_required_attribute + self.cas_required_attribute_value = hs.config.cas_required_attribute_value self.servername = hs.config.server_name def on_GET(self, request): @@ -126,6 +127,19 @@ class LoginRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def do_cas_login(self, cas_response_body): (user, attributes) = self.parse_cas_response(cas_response_body) + + if self.cas_required_attribute is not None: + # If required attribute was not in CAS Response - Forbidden + if self.cas_required_attribute not in attributes: + raise LoginError(401, "Unauthorized", errcode=Codes.UNAUTHORIZED) + + # Also need to check value + if self.cas_required_attribute_value is not None: + actualValue = attributes[self.cas_required_attribute] + # If required attribute value does not match expected - Forbidden + if self.cas_required_attribute_value != actualValue: + raise LoginError(401, "Unauthorized", errcode=Codes.UNAUTHORIZED) + user_id = UserID.create(user, self.hs.hostname).to_string() auth_handler = self.handlers.auth_handler user_exists = yield auth_handler.does_user_exist(user_id) -- cgit 1.5.1 From 01a5f1991c8e54d0762cf1647c941d00c938f994 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Mon, 12 Oct 2015 14:43:17 +0100 Subject: Support multiple required attributes in CAS response, and in a nicer config format too --- synapse/config/cas.py | 19 ++++--------------- synapse/rest/client/v1/login.py | 13 ++++++------- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/synapse/config/cas.py b/synapse/config/cas.py index 4d1dd8cc7b..e884d03fe6 100644 --- a/synapse/config/cas.py +++ b/synapse/config/cas.py @@ -27,28 +27,17 @@ class CasConfig(Config): if cas_config: self.cas_enabled = True self.cas_server_url = cas_config["server_url"] - - if "required_attribute" in cas_config: - self.cas_required_attribute = cas_config["required_attribute"] - else: - self.cas_required_attribute = None - - if "required_attribute_value" in cas_config: - self.cas_required_attribute_value = cas_config["required_attribute_value"] - else: - self.cas_required_attribute_value = None - + self.cas_required_attributes = cas_config.get("required_attributes", None) else: self.cas_enabled = False self.cas_server_url = None - self.cas_required_attribute = None - self.cas_required_attribute_value = None + self.cas_required_attributes = {} def default_config(self, config_dir_path, server_name, **kwargs): return """ # Enable CAS for registration and login. #cas_config: # server_url: "https://cas-server.com" - # #required_attribute: something - # #required_attribute_value: true + # #required_attributes: + # # name: value """ diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 1e62beaff8..84774e61aa 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -46,8 +46,7 @@ class LoginRestServlet(ClientV1RestServlet): self.saml2_enabled = hs.config.saml2_enabled self.cas_enabled = hs.config.cas_enabled self.cas_server_url = hs.config.cas_server_url - self.cas_required_attribute = hs.config.cas_required_attribute - self.cas_required_attribute_value = hs.config.cas_required_attribute_value + self.cas_required_attributes = hs.config.cas_required_attributes self.servername = hs.config.server_name def on_GET(self, request): @@ -128,16 +127,16 @@ class LoginRestServlet(ClientV1RestServlet): def do_cas_login(self, cas_response_body): (user, attributes) = self.parse_cas_response(cas_response_body) - if self.cas_required_attribute is not None: + for required_attribute in self.cas_required_attributes: # If required attribute was not in CAS Response - Forbidden - if self.cas_required_attribute not in attributes: + if required_attribute not in attributes: raise LoginError(401, "Unauthorized", errcode=Codes.UNAUTHORIZED) # Also need to check value - if self.cas_required_attribute_value is not None: - actualValue = attributes[self.cas_required_attribute] + if self.cas_required_attributes[required_attribute] is not None: + actualValue = attributes[required_attribute] # If required attribute value does not match expected - Forbidden - if self.cas_required_attribute_value != actualValue: + if self.cas_required_attributes[required_attribute] != actualValue: raise LoginError(401, "Unauthorized", errcode=Codes.UNAUTHORIZED) user_id = UserID.create(user, self.hs.hostname).to_string() -- cgit 1.5.1 From 7f8fdc9814571723bfc120e43c6d21cde1c660a4 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Mon, 12 Oct 2015 14:45:24 +0100 Subject: Remove not required parenthesis --- synapse/rest/client/v1/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 84774e61aa..8facb00126 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -125,7 +125,7 @@ class LoginRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def do_cas_login(self, cas_response_body): - (user, attributes) = self.parse_cas_response(cas_response_body) + user, attributes = self.parse_cas_response(cas_response_body) for required_attribute in self.cas_required_attributes: # If required attribute was not in CAS Response - Forbidden -- cgit 1.5.1 From ab7f9bb861791b9415d80f0e71d7b4b867b0a445 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Mon, 12 Oct 2015 14:58:59 +0100 Subject: Default cas_required_attributes to empty dictionary --- synapse/config/cas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/cas.py b/synapse/config/cas.py index e884d03fe6..d268680729 100644 --- a/synapse/config/cas.py +++ b/synapse/config/cas.py @@ -27,7 +27,7 @@ class CasConfig(Config): if cas_config: self.cas_enabled = True self.cas_server_url = cas_config["server_url"] - self.cas_required_attributes = cas_config.get("required_attributes", None) + self.cas_required_attributes = cas_config.get("required_attributes", {}) else: self.cas_enabled = False self.cas_server_url = None -- cgit 1.5.1 From 83b464e4f70fbfcc338b0c3533359a8c53890cdc Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Mon, 12 Oct 2015 15:05:34 +0100 Subject: Unpack dictionary in for loop for nicer syntax --- synapse/rest/client/v1/login.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 8facb00126..c92dedcc0f 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -127,16 +127,16 @@ class LoginRestServlet(ClientV1RestServlet): def do_cas_login(self, cas_response_body): user, attributes = self.parse_cas_response(cas_response_body) - for required_attribute in self.cas_required_attributes: + for required_attribute, required_value in self.cas_required_attributes.items(): # If required attribute was not in CAS Response - Forbidden if required_attribute not in attributes: raise LoginError(401, "Unauthorized", errcode=Codes.UNAUTHORIZED) # Also need to check value - if self.cas_required_attributes[required_attribute] is not None: - actualValue = attributes[required_attribute] + if required_value is not None: + actual_value = attributes[required_attribute] # If required attribute value does not match expected - Forbidden - if self.cas_required_attributes[required_attribute] != actualValue: + if required_value != actual_value: raise LoginError(401, "Unauthorized", errcode=Codes.UNAUTHORIZED) user_id = UserID.create(user, self.hs.hostname).to_string() -- cgit 1.5.1 From 927004e34905d4ad6a69576ee1799fe8019d8985 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Oct 2015 15:06:14 +0100 Subject: Remove unused room_id parameter --- synapse/handlers/federation.py | 2 +- synapse/handlers/message.py | 10 ++++----- synapse/handlers/search.py | 50 +++++++++++++++++++++++++++++++++++++++++- synapse/handlers/sync.py | 2 +- synapse/storage/state.py | 11 +++++----- 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3882ba79ed..a710bdcfdb 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -242,7 +242,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _filter_events_for_server(self, server_name, room_id, events): event_to_state = yield self.store.get_state_for_events( - room_id, frozenset(e.event_id for e in events), + frozenset(e.event_id for e in events), types=( (EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, None), diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 30949ff7a6..d2f0892f7a 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -164,7 +164,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def _filter_events_for_client(self, user_id, room_id, events): event_id_to_state = yield self.store.get_state_for_events( - room_id, frozenset(e.event_id for e in events), + frozenset(e.event_id for e in events), types=( (EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, user_id), @@ -290,7 +290,7 @@ class MessageHandler(BaseHandler): elif member_event.membership == Membership.LEAVE: key = (event_type, state_key) room_state = yield self.store.get_state_for_events( - room_id, [member_event.event_id], [key] + [member_event.event_id], [key] ) data = room_state[member_event.event_id].get(key) @@ -314,7 +314,7 @@ class MessageHandler(BaseHandler): room_state = yield self.state_handler.get_current_state(room_id) elif member_event.membership == Membership.LEAVE: room_state = yield self.store.get_state_for_events( - room_id, [member_event.event_id], None + [member_event.event_id], None ) room_state = room_state[member_event.event_id] @@ -403,7 +403,7 @@ class MessageHandler(BaseHandler): elif event.membership == Membership.LEAVE: room_end_token = "s%d" % (event.stream_ordering,) deferred_room_state = self.store.get_state_for_events( - event.room_id, [event.event_id], None + [event.event_id], None ) deferred_room_state.addCallback( lambda states: states[event.event_id] @@ -496,7 +496,7 @@ class MessageHandler(BaseHandler): def _room_initial_sync_parted(self, user_id, room_id, pagin_config, member_event): room_state = yield self.store.get_state_for_events( - member_event.room_id, [member_event.event_id], None + [member_event.event_id], None ) room_state = room_state[member_event.event_id] diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 9dc474aa56..71182a8fe0 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -17,7 +17,9 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.constants import KnownRoomEventKeys, SearchConstraintTypes +from synapse.api.constants import ( + EventTypes, KnownRoomEventKeys, Membership, SearchConstraintTypes +) from synapse.api.errors import SynapseError from synapse.events.utils import serialize_event @@ -71,6 +73,52 @@ class SearchHandler(BaseHandler): def __init__(self, hs): super(SearchHandler, self).__init__(hs) + @defer.inlineCallbacks + def _filter_events_for_client(self, user_id, room_id, events): + event_id_to_state = yield self.store.get_state_for_events( + frozenset(e.event_id for e in events), + types=( + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.Member, user_id), + ) + ) + + def allowed(event, state): + if event.type == EventTypes.RoomHistoryVisibility: + return True + + membership_ev = state.get((EventTypes.Member, user_id), None) + if membership_ev: + membership = membership_ev.membership + else: + membership = Membership.LEAVE + + if membership == Membership.JOIN: + return True + + history = state.get((EventTypes.RoomHistoryVisibility, ''), None) + if history: + visibility = history.content.get("history_visibility", "shared") + else: + visibility = "shared" + + if visibility == "public": + return True + elif visibility == "shared": + return True + elif visibility == "joined": + return membership == Membership.JOIN + elif visibility == "invited": + return membership == Membership.INVITE + + return True + + defer.returnValue([ + event + for event in events + if allowed(event, event_id_to_state[event.event_id]) + ]) + @defer.inlineCallbacks def search(self, user, content): constraint_dicts = content["search_categories"]["room_events"]["constraints"] diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9914ff6f9c..a8940de166 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -312,7 +312,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def _filter_events_for_client(self, user_id, room_id, events): event_id_to_state = yield self.store.get_state_for_events( - room_id, frozenset(e.event_id for e in events), + frozenset(e.event_id for e in events), types=( (EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, user_id), diff --git a/synapse/storage/state.py b/synapse/storage/state.py index e935b9443b..acfb322a53 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -54,7 +54,7 @@ class StateStore(SQLBaseStore): defer.returnValue({}) event_to_groups = yield self._get_state_group_for_events( - room_id, event_ids, + event_ids, ) groups = set(event_to_groups.values()) @@ -208,13 +208,12 @@ class StateStore(SQLBaseStore): ) @defer.inlineCallbacks - def get_state_for_events(self, room_id, event_ids, types): + def get_state_for_events(self, event_ids, types): """Given a list of event_ids and type tuples, return a list of state dicts for each event. The state dicts will only have the type/state_keys that are in the `types` list. Args: - room_id (str) event_ids (list) types (list): List of (type, state_key) tuples which are used to filter the state fetched. `state_key` may be None, which matches @@ -225,7 +224,7 @@ class StateStore(SQLBaseStore): The dicts are mappings from (type, state_key) -> state_events """ event_to_groups = yield self._get_state_group_for_events( - room_id, event_ids, + event_ids, ) groups = set(event_to_groups.values()) @@ -251,8 +250,8 @@ class StateStore(SQLBaseStore): ) @cachedList(cache=_get_state_group_for_event.cache, list_name="event_ids", - num_args=2) - def _get_state_group_for_events(self, room_id, event_ids): + num_args=1) + def _get_state_group_for_events(self, event_ids): """Returns mapping event_id -> state_group """ def f(txn): -- cgit 1.5.1 From ca53ad74250d94b8c9b6581e6cedef0a29520fc2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Oct 2015 15:52:55 +0100 Subject: Filter events to only thsoe that the user is allowed to see --- synapse/handlers/search.py | 16 ++++++++++------ synapse/storage/search.py | 14 +++++++------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 71182a8fe0..49b786dadb 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -74,7 +74,7 @@ class SearchHandler(BaseHandler): super(SearchHandler, self).__init__(hs) @defer.inlineCallbacks - def _filter_events_for_client(self, user_id, room_id, events): + def _filter_events_for_client(self, user_id, events): event_id_to_state = yield self.store.get_state_for_events( frozenset(e.event_id for e in events), types=( @@ -139,16 +139,20 @@ class SearchHandler(BaseHandler): # TODO(paul): work out why because I really don't think it should room_ids = set(r.room_id for r in rooms) - res = yield self.store.search_msgs(room_ids, constraints) + rank_map, event_map = yield self.store.search_msgs(room_ids, constraints) + + allowed_events = yield self._filter_events_for_client( + user.to_string(), event_map.values() + ) time_now = self.clock.time_msec() results = { - r["result"].event_id: { - "rank": r["rank"], - "result": serialize_event(r["result"], time_now) + e.event_id: { + "rank": rank_map[e.event_id], + "result": serialize_event(e, time_now) } - for r in res + for e in allowed_events } logger.info("returning: %r", results) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index e66b5f9edc..238df38440 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -70,11 +70,11 @@ class SearchStore(SQLBaseStore): for ev in events } - defer.returnValue([ + defer.returnValue(( { - "rank": r["rank"], - "result": event_map[r["event_id"]] - } - for r in results - if r["event_id"] in event_map - ]) + r["event_id"]: r["rank"] + for r in results + if r["event_id"] in event_map + }, + event_map + )) -- cgit 1.5.1 From 739464fbc5dc328001fcc71e327938229c836204 Mon Sep 17 00:00:00 2001 From: Steven Hammerton Date: Mon, 12 Oct 2015 16:02:17 +0100 Subject: Add a comment to clarify why we split on closing curly brace when reading CAS attribute tags --- synapse/rest/client/v1/login.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index c92dedcc0f..2e3e4f39f3 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -177,6 +177,11 @@ class LoginRestServlet(ClientV1RestServlet): if child.tag.endswith("attributes"): attributes = {} for attribute in child: + # ElementTree library expands the namespace in attribute tags + # to the full URL of the namespace. + # See (https://docs.python.org/2/library/xml.etree.elementtree.html) + # We don't care about namespace here and it will always be encased in + # curly braces, so we remove them. if "}" in attribute.tag: attributes[attribute.tag.split("}")[1]] = attribute.text else: -- cgit 1.5.1 From 586beb8318bd259581918a8b47f5981f0b90b7e9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 12 Oct 2015 16:54:58 +0100 Subject: Update the filters to match the latest spec. Apply the filter the 'timeline' and 'ephemeral' keys of rooms. Apply the filter to the 'presence' key of a sync response. --- synapse/api/filtering.py | 53 ++++++++++++++++++++++++------------ synapse/handlers/sync.py | 6 ++-- synapse/rest/client/v2_alpha/sync.py | 26 ++++++++---------- tests/api/test_filtering.py | 12 ++++---- 4 files changed, 55 insertions(+), 42 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 2d5431ba60..e79e91e7eb 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -144,17 +144,14 @@ class Filter(object): def ephemeral_limit(self): return self.filter_json.get("room", {}).get("ephemeral", {}).get("limit", 10) - def filter_public_user_data(self, events): - return self._filter_on_key(events, ["public_user_data"]) - - def filter_private_user_data(self, events): - return self._filter_on_key(events, ["private_user_data"]) + def filter_presence(self, events): + return self._filter_on_key(events, ["presence"]) def filter_room_state(self, events): return self._filter_on_key(events, ["room", "state"]) - def filter_room_events(self, events): - return self._filter_on_key(events, ["room", "events"]) + def filter_room_timeline(self, events): + return self._filter_on_key(events, ["room", "timeline"]) def filter_room_ephemeral(self, events): return self._filter_on_key(events, ["room", "ephemeral"]) @@ -178,11 +175,34 @@ class Filter(object): return [e for e in events if self._passes_definition(definition, e)] def _passes_definition(self, definition, event): + """Check if the event passes the filter definition + Args: + definition(dict): The filter definition to check against + event(dict or Event): The event to check + Returns: + True if the event passes the filter in the definition + """ + if type(event) is dict: + room_id = event.get("room_id") + sender = event.get("sender") + event_type = event["type"] + else: + room_id = getattr(event, "room_id", None) + sender = getattr(event, "sender", None) + event_type = event.type + return self._event_passes_definition( + definition, room_id, sender, event_type + ) + + def _event_passes_definition(self, definition, room_id, sender, + event_type): """Check if the event passes through the given definition. Args: definition(dict): The definition to check against. - event(Event): The event to check. + room_id(str): The id of the room this event is in or None. + sender(str): The sender of the event + event_type(str): The type of the event. Returns: True if the event passes through the filter. """ @@ -194,8 +214,7 @@ class Filter(object): # and 'not_types' then it is treated as only being in 'not_types') # room checks - if hasattr(event, "room_id"): - room_id = event.room_id + if room_id is not None: allow_rooms = definition.get("rooms", None) reject_rooms = definition.get("not_rooms", None) if reject_rooms and room_id in reject_rooms: @@ -204,9 +223,7 @@ class Filter(object): return False # sender checks - if hasattr(event, "sender"): - # Should we be including event.state_key for some event types? - sender = event.sender + if sender is not None: allow_senders = definition.get("senders", None) reject_senders = definition.get("not_senders", None) if reject_senders and sender in reject_senders: @@ -217,12 +234,12 @@ class Filter(object): # type checks if "not_types" in definition: for def_type in definition["not_types"]: - if self._event_matches_type(event, def_type): + if self._event_matches_type(event_type, def_type): return False if "types" in definition: included = False for def_type in definition["types"]: - if self._event_matches_type(event, def_type): + if self._event_matches_type(event_type, def_type): included = True break if not included: @@ -230,9 +247,9 @@ class Filter(object): return True - def _event_matches_type(self, event, def_type): + def _event_matches_type(self, event_type, def_type): if def_type.endswith("*"): type_prefix = def_type[:-1] - return event.type.startswith(type_prefix) + return event_type.startswith(type_prefix) else: - return event.type == def_type + return event_type == def_type diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 76cca7c621..edc728ece1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -277,7 +277,7 @@ class SyncHandler(BaseHandler): for room_id in room_ids: room_sync = yield self.incremental_sync_with_gap_for_room( room_id, sync_config, since_token, now_token, - published_room_ids, typing_by_room + typing_by_room ) if room_sync: rooms.append(room_sync) @@ -355,7 +355,7 @@ class SyncHandler(BaseHandler): ) (room_key, _) = keys end_key = "s" + room_key.split('-')[-1] - loaded_recents = sync_config.filter.filter_room_events(events) + loaded_recents = sync_config.filter.filter_room_timeline(events) loaded_recents = yield self._filter_events_for_client( sync_config.user.to_string(), room_id, loaded_recents, ) @@ -381,7 +381,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, - published_room_ids, typing_by_room): + typing_by_room): """ Get the incremental delta needed to bring the client up to date for the room. Gives the client the most recent events and the changes to state. diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 3348b46c14..1223a4a7f6 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -46,11 +46,6 @@ class SyncRestServlet(RestServlet): { "next_batch": // batch token for the next /sync "presence": // presence data for the user. - "invited": [], // Ids of invited rooms being updated. - "joined": [], // Ids of joined rooms being updated. - "archived": [] // Ids of archived rooms being updated. - } - } "rooms": { "joined": { // Joined rooms being updated. "${room_id}": { // Id of the room being updated @@ -67,8 +62,8 @@ class SyncRestServlet(RestServlet): "ephemeral": {"events": []} // list of event objects } }, - "invited": {}, // Ids of invited rooms being updated. - "archived": {} // Ids of archived rooms being updated. + "invited": {}, // Invited rooms being updated. + "archived": {} // Archived rooms being updated. } } """ @@ -151,9 +146,9 @@ class SyncRestServlet(RestServlet): formatted = [] for event in events: event = copy.deepcopy(event) - event['sender'] = event['content'].pop('user_id'); + event['sender'] = event['content'].pop('user_id') formatted.append(event) - return {"events": formatted} + return {"events": filter.filter_presence(formatted)} def encode_rooms(self, rooms, filter, time_now, token_id): joined = {} @@ -172,9 +167,10 @@ class SyncRestServlet(RestServlet): def encode_room(room, filter, time_now, token_id): event_map = {} state_events = filter.filter_room_state(room.state) - recent_events = filter.filter_room_events(room.timeline.events) + timeline_events = filter.filter_room_timeline(room.timeline.events) + ephemeral_events = filter.filter_room_ephemeral(room.ephemeral) state_event_ids = [] - recent_event_ids = [] + timeline_event_ids = [] for event in state_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( @@ -183,22 +179,22 @@ class SyncRestServlet(RestServlet): ) state_event_ids.append(event.event_id) - for event in recent_events: + for event in timeline_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( event, time_now, token_id=token_id, event_format=format_event_for_client_v2_without_event_id, ) - recent_event_ids.append(event.event_id) + timeline_event_ids.append(event.event_id) result = { "event_map": event_map, "timeline": { - "events": recent_event_ids, + "events": timeline_event_ids, "prev_batch": room.timeline.prev_batch.to_string(), "limited": room.timeline.limited, }, "state": {"events": state_event_ids}, - "ephemeral": {"events": room.ephemeral}, + "ephemeral": {"events": ephemeral_events}, } return result diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 65b2f590c8..6942cdac51 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -345,9 +345,9 @@ class FilteringTestCase(unittest.TestCase): ) @defer.inlineCallbacks - def test_filter_public_user_data_match(self): + def test_filter_presence_match(self): user_filter_json = { - "public_user_data": { + "presence": { "types": ["m.*"] } } @@ -368,13 +368,13 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - results = user_filter.filter_public_user_data(events=events) + results = user_filter.filter_presence(events=events) self.assertEquals(events, results) @defer.inlineCallbacks - def test_filter_public_user_data_no_match(self): + def test_filter_presence_no_match(self): user_filter_json = { - "public_user_data": { + "presence": { "types": ["m.*"] } } @@ -395,7 +395,7 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - results = user_filter.filter_public_user_data(events=events) + results = user_filter.filter_presence(events=events) self.assertEquals([], results) @defer.inlineCallbacks -- cgit 1.5.1 From 956509dfecccca944d89bc9e9f002e5039cf81fc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 13 Oct 2015 10:24:51 +0100 Subject: Start spliting out the rooms into joined and invited in v2 sync --- synapse/handlers/sync.py | 58 +++++++++++++++++++++++++----------- synapse/rest/client/v2_alpha/sync.py | 18 +++++------ 2 files changed, 49 insertions(+), 27 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index edc728ece1..e693e7c80e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -46,7 +46,7 @@ class TimelineBatch(collections.namedtuple("TimelineBatch", [ return bool(self.events) -class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ +class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [ "room_id", "timeline", "state", @@ -61,10 +61,24 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ return bool(self.timeline or self.state or self.ephemeral) +class InvitedSyncResult(collections.namedtuple("InvitedSyncResult", [ + "room_id", + "invite_state", +])): + __slots__ = [] + + def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if room needs to be part of the sync result. + """ + return bool(self.invite_state) + + class SyncResult(collections.namedtuple("SyncResult", [ "next_batch", # Token for the next sync "presence", # List of presence events for the user. - "rooms", # RoomSyncResult for each room. + "joined", # JoinedSyncResult for each joined room. + "invited", # InvitedSyncResult for each invited room. ])): __slots__ = [] @@ -151,24 +165,31 @@ class SyncHandler(BaseHandler): membership_list=[Membership.INVITE, Membership.JOIN] ) - rooms = [] + joined = [] for event in room_list: - room_sync = yield self.initial_sync_for_room( - event.room_id, sync_config, now_token, - ) - rooms.append(room_sync) + if event.membership == Membership.JOIN: + room_sync = yield self.initial_sync_for_room( + event.room_id, sync_config, now_token, + ) + joined.append(room_sync) + elif event.membership == Membership.INVITE: + invited.append(InvitedSyncResult( + room_id=event.room_id, + invited_state=[event], + ) defer.returnValue(SyncResult( presence=presence, - rooms=rooms, + joined=joined, + invited=[], next_batch=now_token, )) @defer.inlineCallbacks - def initial_sync_for_room(self, room_id, sync_config, now_token): + def initial_sync_for_joined_room(self, room_id, sync_config, now_token): """Sync a room for a client which is starting without any state Returns: - A Deferred RoomSyncResult. + A Deferred JoinedSyncResult. """ batch = yield self.load_filtered_recents( @@ -180,7 +201,7 @@ class SyncHandler(BaseHandler): ) current_state_events = current_state.values() - defer.returnValue(RoomSyncResult( + defer.returnValue(JoinedSyncResult( room_id=room_id, timeline=batch, state=current_state_events, @@ -239,7 +260,7 @@ class SyncHandler(BaseHandler): limit=timeline_limit + 1, ) - rooms = [] + joined = [] if len(room_events) <= timeline_limit: # There is no gap in any of the rooms. Therefore we can just # partition the new events by room and return them. @@ -261,7 +282,7 @@ class SyncHandler(BaseHandler): sync_config, room_id, state ) - room_sync = RoomSyncResult( + room_sync = JoinedSyncResult( room_id=room_id, timeline=TimelineBatch( events=recents, @@ -272,7 +293,7 @@ class SyncHandler(BaseHandler): ephemeral=typing_by_room.get(room_id, []) ) if room_sync: - rooms.append(room_sync) + joined.append(room_sync) else: for room_id in room_ids: room_sync = yield self.incremental_sync_with_gap_for_room( @@ -280,11 +301,12 @@ class SyncHandler(BaseHandler): typing_by_room ) if room_sync: - rooms.append(room_sync) + joined.append(room_sync) defer.returnValue(SyncResult( presence=presence, - rooms=rooms, + joined=joined, + invited=[], next_batch=now_token, )) @@ -386,7 +408,7 @@ class SyncHandler(BaseHandler): the room. Gives the client the most recent events and the changes to state. Returns: - A Deferred RoomSyncResult + A Deferred JoinedSyncResult """ # TODO(mjark): Check for redactions we might have missed. @@ -418,7 +440,7 @@ class SyncHandler(BaseHandler): sync_config, room_id, state_events_delta ) - room_sync = RoomSyncResult( + room_sync = JoinedSyncResult( room_id=room_id, timeline=batch, state=state_events_delta, diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 1223a4a7f6..9b87879f51 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -128,15 +128,19 @@ class SyncRestServlet(RestServlet): time_now = self.clock.time_msec() - rooms = self.encode_rooms( - sync_result.rooms, filter, time_now, token_id + joined = self.encode_joined( + sync_result.joined, filter, time_now, token_id ) response_content = { "presence": self.encode_presence( sync_result.presence, filter, time_now ), - "rooms": rooms, + "rooms": { + "joined": joined, + "invited": {}, + "archived": {}, + }, "next_batch": sync_result.next_batch.to_string(), } @@ -150,18 +154,14 @@ class SyncRestServlet(RestServlet): formatted.append(event) return {"events": filter.filter_presence(formatted)} - def encode_rooms(self, rooms, filter, time_now, token_id): + def encode_joined(self, rooms, filter, time_now, token_id): joined = {} for room in rooms: joined[room.room_id] = self.encode_room( room, filter, time_now, token_id ) - return { - "joined": joined, - "invited": {}, - "archived": {}, - } + return joined @staticmethod def encode_room(room, filter, time_now, token_id): -- cgit 1.5.1 From 1a40afa75693f0c2ae3b2eaac62ff9ca6bb02488 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Oct 2015 10:36:25 +0100 Subject: Add sqlite schema --- synapse/storage/schema/delta/24/fts.py | 69 +++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/synapse/storage/schema/delta/24/fts.py b/synapse/storage/schema/delta/24/fts.py index 05f1605fdd..a806f4b8d3 100644 --- a/synapse/storage/schema/delta/24/fts.py +++ b/synapse/storage/schema/delta/24/fts.py @@ -15,7 +15,9 @@ import logging from synapse.storage import get_statements -from synapse.storage.engines import PostgresEngine +from synapse.storage.engines import PostgresEngine, Sqlite3Engine + +import ujson logger = logging.getLogger(__name__) @@ -46,13 +48,70 @@ INSERT INTO event_search SELECT CREATE INDEX event_search_fts_idx ON event_search USING gin(vector); CREATE INDEX event_search_ev_idx ON event_search(event_id); +CREATE INDEX event_search_ev_ridx ON event_search(room_id); """ +SQLITE_TABLE = ( + "CREATE VIRTUAL TABLE event_search USING fts3 ( event_id, room_id, key, value)" +) +SQLITE_INDEX = "CREATE INDEX event_search_ev_idx ON event_search(event_id)" + + def run_upgrade(cur, database_engine, *args, **kwargs): - if not isinstance(database_engine, PostgresEngine): - # We only support FTS for postgres currently. + if isinstance(database_engine, PostgresEngine): + for statement in get_statements(POSTGRES_SQL.splitlines()): + cur.execute(statement) return - for statement in get_statements(POSTGRES_SQL.splitlines()): - cur.execute(statement) + if isinstance(database_engine, Sqlite3Engine): + cur.execute(SQLITE_TABLE) + + rowid = -1 + while True: + cur.execute( + "SELECT rowid, json FROM event_json" + " WHERE rowid > ?" + " ORDER BY rowid ASC LIMIT 100", + (rowid,) + ) + + res = cur.fetchall() + + if not res: + break + + events = [ + ujson.loads(js) + for _, js in res + ] + + rowid = max(rid for rid, _ in res) + + rows = [] + for ev in events: + if ev["type"] == "m.room.message": + rows.append(( + ev["event_id"], ev["room_id"], "content.body", + ev["content"]["body"] + )) + if ev["type"] == "m.room.name": + rows.append(( + ev["event_id"], ev["room_id"], "content.name", + ev["content"]["name"] + )) + if ev["type"] == "m.room.topic": + rows.append(( + ev["event_id"], ev["room_id"], "content.topic", + ev["content"]["topic"] + )) + + if rows: + logger.info(rows) + cur.executemany( + "INSERT INTO event_search (event_id, room_id, key, value)" + " VALUES (?,?,?,?)", + rows + ) + + # cur.execute(SQLITE_INDEX) -- cgit 1.5.1 From 30c2783d2f2983764738383d73c378ec5dc61279 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Oct 2015 10:36:36 +0100 Subject: Search left rooms too --- synapse/handlers/search.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 49b786dadb..d5c395061c 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -131,12 +131,9 @@ class SearchHandler(BaseHandler): raise SynapseError(400, "Only one constraint can be FTS") fts = True - rooms = yield self.store.get_rooms_for_user( - user.to_string(), + rooms = yield self.store.get_rooms_for_user_where_membership_is( + user.to_string(), membership_list=[Membership.JOIN, Membership.LEAVE], ) - - # For some reason the list of events contains duplicates - # TODO(paul): work out why because I really don't think it should room_ids = set(r.room_id for r in rooms) rank_map, event_map = yield self.store.search_msgs(room_ids, constraints) -- cgit 1.5.1 From ab9cf732585244781ba67f4bb4c235ded3d4661a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 13 Oct 2015 11:03:48 +0100 Subject: Include invited rooms in the initial sync --- synapse/handlers/sync.py | 16 ++++++---------- synapse/rest/client/v2_alpha/sync.py | 21 ++++++++++++++++++++- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index e693e7c80e..574412d6b5 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -63,16 +63,10 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [ class InvitedSyncResult(collections.namedtuple("InvitedSyncResult", [ "room_id", - "invite_state", + "invite", ])): __slots__ = [] - def __nonzero__(self): - """Make the result appear empty if there are no updates. This is used - to tell if room needs to be part of the sync result. - """ - return bool(self.invite_state) - class SyncResult(collections.namedtuple("SyncResult", [ "next_batch", # Token for the next sync @@ -166,6 +160,7 @@ class SyncHandler(BaseHandler): ) joined = [] + invited = [] for event in room_list: if event.membership == Membership.JOIN: room_sync = yield self.initial_sync_for_room( @@ -173,15 +168,16 @@ class SyncHandler(BaseHandler): ) joined.append(room_sync) elif event.membership == Membership.INVITE: + invite = yield self.store.get_event(event.event_id) invited.append(InvitedSyncResult( room_id=event.room_id, - invited_state=[event], - ) + invite=invite, + )) defer.returnValue(SyncResult( presence=presence, joined=joined, - invited=[], + invited=invited, next_batch=now_token, )) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 9b87879f51..399df9e772 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -132,13 +132,17 @@ class SyncRestServlet(RestServlet): sync_result.joined, filter, time_now, token_id ) + invited = self.encode_invited( + sync_result.invited, filter, time_now, token_id + ) + response_content = { "presence": self.encode_presence( sync_result.presence, filter, time_now ), "rooms": { "joined": joined, - "invited": {}, + "invited": invited, "archived": {}, }, "next_batch": sync_result.next_batch.to_string(), @@ -163,6 +167,21 @@ class SyncRestServlet(RestServlet): return joined + def encode_invited(self, rooms, filter, time_now, token_id): + invited = {} + for room in rooms: + invite = serialize_event( + room.invite, time_now, token_id=token_id, + event_format=format_event_for_client_v2_without_event_id, + ) + invited_state = invite.get("unsigned", {}).pop("invite_room_state", []) + invited_state.append(invite) + invited[room.room_id] = { + "invite_state": { "events": invited_state } + } + + return invited + @staticmethod def encode_room(room, filter, time_now, token_id): event_map = {} -- cgit 1.5.1 From 40b6a5aad1309fed9d1e32be387798dd46b2cf4f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Oct 2015 11:38:48 +0100 Subject: Split out the schema preparation and update logic into its own module --- synapse/storage/__init__.py | 378 +--------------------------------- synapse/storage/_schema_prepare.py | 395 ++++++++++++++++++++++++++++++++++++ synapse/storage/engines/postgres.py | 2 +- synapse/storage/engines/sqlite3.py | 4 +- 4 files changed, 402 insertions(+), 377 deletions(-) create mode 100644 synapse/storage/_schema_prepare.py diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 340e59afcb..4be629bff8 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -41,23 +41,16 @@ from .end_to_end_keys import EndToEndKeyStore from .receipts import ReceiptsStore +from ._schema_prepare import UpgradeDatabaseException + +__all__ = [UpgradeDatabaseException] -import fnmatch -import imp import logging -import os -import re logger = logging.getLogger(__name__) -# Remember to update this number every time a change is made to database -# schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 24 - -dir_path = os.path.abspath(os.path.dirname(__file__)) - # Number of msec of granularity to store the user IP 'last seen' time. Smaller # times give more inserts into the database even for readonly API hits # 120 seconds == 2 minutes @@ -158,371 +151,6 @@ class DataStore(RoomMemberStore, RoomStore, ) -def read_schema(path): - """ Read the named database schema. - - Args: - path: Path of the database schema. - Returns: - A string containing the database schema. - """ - with open(path) as schema_file: - return schema_file.read() - - -class PrepareDatabaseException(Exception): - pass - - -class UpgradeDatabaseException(PrepareDatabaseException): - pass - - -def prepare_database(db_conn, database_engine): - """Prepares a database for usage. Will either create all necessary tables - or upgrade from an older schema version. - """ - try: - cur = db_conn.cursor() - version_info = _get_or_create_schema_state(cur, database_engine) - - if version_info: - user_version, delta_files, upgraded = version_info - _upgrade_existing_database( - cur, user_version, delta_files, upgraded, database_engine - ) - else: - _setup_new_database(cur, database_engine) - - # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) - - cur.close() - db_conn.commit() - except: - db_conn.rollback() - raise - - -def _setup_new_database(cur, database_engine): - """Sets up the database by finding a base set of "full schemas" and then - applying any necessary deltas. - - The "full_schemas" directory has subdirectories named after versions. This - function searches for the highest version less than or equal to - `SCHEMA_VERSION` and executes all .sql files in that directory. - - The function will then apply all deltas for all versions after the base - version. - - Example directory structure: - - schema/ - delta/ - ... - full_schemas/ - 3/ - test.sql - ... - 11/ - foo.sql - bar.sql - ... - - In the example foo.sql and bar.sql would be run, and then any delta files - for versions strictly greater than 11. - """ - current_dir = os.path.join(dir_path, "schema", "full_schemas") - directory_entries = os.listdir(current_dir) - - valid_dirs = [] - pattern = re.compile(r"^\d+(\.sql)?$") - for filename in directory_entries: - match = pattern.match(filename) - abs_path = os.path.join(current_dir, filename) - if match and os.path.isdir(abs_path): - ver = int(match.group(0)) - if ver <= SCHEMA_VERSION: - valid_dirs.append((ver, abs_path)) - else: - logger.warn("Unexpected entry in 'full_schemas': %s", filename) - - if not valid_dirs: - raise PrepareDatabaseException( - "Could not find a suitable base set of full schemas" - ) - - max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0]) - - logger.debug("Initialising schema v%d", max_current_ver) - - directory_entries = os.listdir(sql_dir) - - for filename in fnmatch.filter(directory_entries, "*.sql"): - sql_loc = os.path.join(sql_dir, filename) - logger.debug("Applying schema %s", sql_loc) - executescript(cur, sql_loc) - - cur.execute( - database_engine.convert_param_style( - "INSERT INTO schema_version (version, upgraded)" - " VALUES (?,?)" - ), - (max_current_ver, False,) - ) - - _upgrade_existing_database( - cur, - current_version=max_current_ver, - applied_delta_files=[], - upgraded=False, - database_engine=database_engine, - ) - - -def _upgrade_existing_database(cur, current_version, applied_delta_files, - upgraded, database_engine): - """Upgrades an existing database. - - Delta files can either be SQL stored in *.sql files, or python modules - in *.py. - - There can be multiple delta files per version. Synapse will keep track of - which delta files have been applied, and will apply any that haven't been - even if there has been no version bump. This is useful for development - where orthogonal schema changes may happen on separate branches. - - Different delta files for the same version *must* be orthogonal and give - the same result when applied in any order. No guarantees are made on the - order of execution of these scripts. - - This is a no-op of current_version == SCHEMA_VERSION. - - Example directory structure: - - schema/ - delta/ - 11/ - foo.sql - ... - 12/ - foo.sql - bar.py - ... - full_schemas/ - ... - - In the example, if current_version is 11, then foo.sql will be run if and - only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in - some arbitrary order. - - Args: - cur (Cursor) - current_version (int): The current version of the schema. - applied_delta_files (list): A list of deltas that have already been - applied. - upgraded (bool): Whether the current version was generated by having - applied deltas or from full schema file. If `True` the function - will never apply delta files for the given `current_version`, since - the current_version wasn't generated by applying those delta files. - """ - - if current_version > SCHEMA_VERSION: - raise ValueError( - "Cannot use this database as it is too " + - "new for the server to understand" - ) - - start_ver = current_version - if not upgraded: - start_ver += 1 - - logger.debug("applied_delta_files: %s", applied_delta_files) - - for v in range(start_ver, SCHEMA_VERSION + 1): - logger.debug("Upgrading schema to v%d", v) - - delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) - - try: - directory_entries = os.listdir(delta_dir) - except OSError: - logger.exception("Could not open delta dir for version %d", v) - raise UpgradeDatabaseException( - "Could not open delta dir for version %d" % (v,) - ) - - directory_entries.sort() - for file_name in directory_entries: - relative_path = os.path.join(str(v), file_name) - logger.debug("Found file: %s", relative_path) - if relative_path in applied_delta_files: - continue - - absolute_path = os.path.join( - dir_path, "schema", "delta", relative_path, - ) - root_name, ext = os.path.splitext(file_name) - if ext == ".py": - # This is a python upgrade module. We need to import into some - # package and then execute its `run_upgrade` function. - module_name = "synapse.storage.v%d_%s" % ( - v, root_name - ) - with open(absolute_path) as python_file: - module = imp.load_source( - module_name, absolute_path, python_file - ) - logger.debug("Running script %s", relative_path) - module.run_upgrade(cur, database_engine) - elif ext == ".pyc": - # Sometimes .pyc files turn up anyway even though we've - # disabled their generation; e.g. from distribution package - # installers. Silently skip it - pass - elif ext == ".sql": - # A plain old .sql file, just read and execute it - logger.debug("Applying schema %s", relative_path) - executescript(cur, absolute_path) - else: - # Not a valid delta file. - logger.warn( - "Found directory entry that did not end in .py or" - " .sql: %s", - relative_path, - ) - continue - - # Mark as done. - cur.execute( - database_engine.convert_param_style( - "INSERT INTO applied_schema_deltas (version, file)" - " VALUES (?,?)", - ), - (v, relative_path) - ) - - cur.execute("DELETE FROM schema_version") - cur.execute( - database_engine.convert_param_style( - "INSERT INTO schema_version (version, upgraded)" - " VALUES (?,?)", - ), - (v, True) - ) - - -def get_statements(f): - statement_buffer = "" - in_comment = False # If we're in a /* ... */ style comment - - for line in f: - line = line.strip() - - if in_comment: - # Check if this line contains an end to the comment - comments = line.split("*/", 1) - if len(comments) == 1: - continue - line = comments[1] - in_comment = False - - # Remove inline block comments - line = re.sub(r"/\*.*\*/", " ", line) - - # Does this line start a comment? - comments = line.split("/*", 1) - if len(comments) > 1: - line = comments[0] - in_comment = True - - # Deal with line comments - line = line.split("--", 1)[0] - line = line.split("//", 1)[0] - - # Find *all* semicolons. We need to treat first and last entry - # specially. - statements = line.split(";") - - # We must prepend statement_buffer to the first statement - first_statement = "%s %s" % ( - statement_buffer.strip(), - statements[0].strip() - ) - statements[0] = first_statement - - # Every entry, except the last, is a full statement - for statement in statements[:-1]: - yield statement.strip() - - # The last entry did *not* end in a semicolon, so we store it for the - # next semicolon we find - statement_buffer = statements[-1].strip() - - -def executescript(txn, schema_path): - with open(schema_path, 'r') as f: - for statement in get_statements(f): - txn.execute(statement) - - -def _get_or_create_schema_state(txn, database_engine): - # Bluntly try creating the schema_version tables. - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - executescript(txn, schema_path) - - txn.execute("SELECT version, upgraded FROM schema_version") - row = txn.fetchone() - current_version = int(row[0]) if row else None - upgraded = bool(row[1]) if row else None - - if current_version: - txn.execute( - database_engine.convert_param_style( - "SELECT file FROM applied_schema_deltas WHERE version >= ?" - ), - (current_version,) - ) - applied_deltas = [d for d, in txn.fetchall()] - return current_version, applied_deltas, upgraded - - return None - - -def prepare_sqlite3_database(db_conn): - """This function should be called before `prepare_database` on sqlite3 - databases. - - Since we changed the way we store the current schema version and handle - updates to schemas, we need a way to upgrade from the old method to the - new. This only affects sqlite databases since they were the only ones - supported at the time. - """ - with db_conn: - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - create_schema = read_schema(schema_path) - db_conn.executescript(create_schema) - - c = db_conn.execute("SELECT * FROM schema_version") - rows = c.fetchall() - c.close() - - if not rows: - c = db_conn.execute("PRAGMA user_version") - row = c.fetchone() - c.close() - - if row and row[0]: - db_conn.execute( - "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", - (row[0], False) - ) - - def are_all_users_on_domain(txn, database_engine, domain): sql = database_engine.convert_param_style( "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" diff --git a/synapse/storage/_schema_prepare.py b/synapse/storage/_schema_prepare.py new file mode 100644 index 0000000000..1ddf55be4d --- /dev/null +++ b/synapse/storage/_schema_prepare.py @@ -0,0 +1,395 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import imp +import logging +import os +import re + + +logger = logging.getLogger(__name__) + + +# Remember to update this number every time a change is made to database +# schema files, so the users will be informed on server restarts. +SCHEMA_VERSION = 24 + +dir_path = os.path.abspath(os.path.dirname(__file__)) + + +def read_schema(path): + """ Read the named database schema. + + Args: + path: Path of the database schema. + Returns: + A string containing the database schema. + """ + with open(path) as schema_file: + return schema_file.read() + + +class PrepareDatabaseException(Exception): + pass + + +class UpgradeDatabaseException(PrepareDatabaseException): + pass + + +def prepare_database(db_conn, database_engine): + """Prepares a database for usage. Will either create all necessary tables + or upgrade from an older schema version. + """ + try: + cur = db_conn.cursor() + version_info = _get_or_create_schema_state(cur, database_engine) + + if version_info: + user_version, delta_files, upgraded = version_info + _upgrade_existing_database( + cur, user_version, delta_files, upgraded, database_engine + ) + else: + _setup_new_database(cur, database_engine) + + # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) + + cur.close() + db_conn.commit() + except: + db_conn.rollback() + raise + + +def _setup_new_database(cur, database_engine): + """Sets up the database by finding a base set of "full schemas" and then + applying any necessary deltas. + + The "full_schemas" directory has subdirectories named after versions. This + function searches for the highest version less than or equal to + `SCHEMA_VERSION` and executes all .sql files in that directory. + + The function will then apply all deltas for all versions after the base + version. + + Example directory structure: + + schema/ + delta/ + ... + full_schemas/ + 3/ + test.sql + ... + 11/ + foo.sql + bar.sql + ... + + In the example foo.sql and bar.sql would be run, and then any delta files + for versions strictly greater than 11. + """ + current_dir = os.path.join(dir_path, "schema", "full_schemas") + directory_entries = os.listdir(current_dir) + + valid_dirs = [] + pattern = re.compile(r"^\d+(\.sql)?$") + for filename in directory_entries: + match = pattern.match(filename) + abs_path = os.path.join(current_dir, filename) + if match and os.path.isdir(abs_path): + ver = int(match.group(0)) + if ver <= SCHEMA_VERSION: + valid_dirs.append((ver, abs_path)) + else: + logger.warn("Unexpected entry in 'full_schemas': %s", filename) + + if not valid_dirs: + raise PrepareDatabaseException( + "Could not find a suitable base set of full schemas" + ) + + max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0]) + + logger.debug("Initialising schema v%d", max_current_ver) + + directory_entries = os.listdir(sql_dir) + + for filename in fnmatch.filter(directory_entries, "*.sql"): + sql_loc = os.path.join(sql_dir, filename) + logger.debug("Applying schema %s", sql_loc) + executescript(cur, sql_loc) + + cur.execute( + database_engine.convert_param_style( + "INSERT INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), + (max_current_ver, False,) + ) + + _upgrade_existing_database( + cur, + current_version=max_current_ver, + applied_delta_files=[], + upgraded=False, + database_engine=database_engine, + ) + + +def _upgrade_existing_database(cur, current_version, applied_delta_files, + upgraded, database_engine): + """Upgrades an existing database. + + Delta files can either be SQL stored in *.sql files, or python modules + in *.py. + + There can be multiple delta files per version. Synapse will keep track of + which delta files have been applied, and will apply any that haven't been + even if there has been no version bump. This is useful for development + where orthogonal schema changes may happen on separate branches. + + Different delta files for the same version *must* be orthogonal and give + the same result when applied in any order. No guarantees are made on the + order of execution of these scripts. + + This is a no-op of current_version == SCHEMA_VERSION. + + Example directory structure: + + schema/ + delta/ + 11/ + foo.sql + ... + 12/ + foo.sql + bar.py + ... + full_schemas/ + ... + + In the example, if current_version is 11, then foo.sql will be run if and + only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in + some arbitrary order. + + Args: + cur (Cursor) + current_version (int): The current version of the schema. + applied_delta_files (list): A list of deltas that have already been + applied. + upgraded (bool): Whether the current version was generated by having + applied deltas or from full schema file. If `True` the function + will never apply delta files for the given `current_version`, since + the current_version wasn't generated by applying those delta files. + """ + + if current_version > SCHEMA_VERSION: + raise ValueError( + "Cannot use this database as it is too " + + "new for the server to understand" + ) + + start_ver = current_version + if not upgraded: + start_ver += 1 + + logger.debug("applied_delta_files: %s", applied_delta_files) + + for v in range(start_ver, SCHEMA_VERSION + 1): + logger.debug("Upgrading schema to v%d", v) + + delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) + + try: + directory_entries = os.listdir(delta_dir) + except OSError: + logger.exception("Could not open delta dir for version %d", v) + raise UpgradeDatabaseException( + "Could not open delta dir for version %d" % (v,) + ) + + directory_entries.sort() + for file_name in directory_entries: + relative_path = os.path.join(str(v), file_name) + logger.debug("Found file: %s", relative_path) + if relative_path in applied_delta_files: + continue + + absolute_path = os.path.join( + dir_path, "schema", "delta", relative_path, + ) + root_name, ext = os.path.splitext(file_name) + if ext == ".py": + # This is a python upgrade module. We need to import into some + # package and then execute its `run_upgrade` function. + module_name = "synapse.storage.v%d_%s" % ( + v, root_name + ) + with open(absolute_path) as python_file: + module = imp.load_source( + module_name, absolute_path, python_file + ) + logger.debug("Running script %s", relative_path) + module.run_upgrade(cur, database_engine) + elif ext == ".pyc": + # Sometimes .pyc files turn up anyway even though we've + # disabled their generation; e.g. from distribution package + # installers. Silently skip it + pass + elif ext == ".sql": + # A plain old .sql file, just read and execute it + logger.debug("Applying schema %s", relative_path) + executescript(cur, absolute_path) + else: + # Not a valid delta file. + logger.warn( + "Found directory entry that did not end in .py or" + " .sql: %s", + relative_path, + ) + continue + + # Mark as done. + cur.execute( + database_engine.convert_param_style( + "INSERT INTO applied_schema_deltas (version, file)" + " VALUES (?,?)", + ), + (v, relative_path) + ) + + cur.execute("DELETE FROM schema_version") + cur.execute( + database_engine.convert_param_style( + "INSERT INTO schema_version (version, upgraded)" + " VALUES (?,?)", + ), + (v, True) + ) + + +def get_statements(f): + statement_buffer = "" + in_comment = False # If we're in a /* ... */ style comment + + for line in f: + line = line.strip() + + if in_comment: + # Check if this line contains an end to the comment + comments = line.split("*/", 1) + if len(comments) == 1: + continue + line = comments[1] + in_comment = False + + # Remove inline block comments + line = re.sub(r"/\*.*\*/", " ", line) + + # Does this line start a comment? + comments = line.split("/*", 1) + if len(comments) > 1: + line = comments[0] + in_comment = True + + # Deal with line comments + line = line.split("--", 1)[0] + line = line.split("//", 1)[0] + + # Find *all* semicolons. We need to treat first and last entry + # specially. + statements = line.split(";") + + # We must prepend statement_buffer to the first statement + first_statement = "%s %s" % ( + statement_buffer.strip(), + statements[0].strip() + ) + statements[0] = first_statement + + # Every entry, except the last, is a full statement + for statement in statements[:-1]: + yield statement.strip() + + # The last entry did *not* end in a semicolon, so we store it for the + # next semicolon we find + statement_buffer = statements[-1].strip() + + +def executescript(txn, schema_path): + with open(schema_path, 'r') as f: + for statement in get_statements(f): + txn.execute(statement) + + +def _get_or_create_schema_state(txn, database_engine): + # Bluntly try creating the schema_version tables. + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + executescript(txn, schema_path) + + txn.execute("SELECT version, upgraded FROM schema_version") + row = txn.fetchone() + current_version = int(row[0]) if row else None + upgraded = bool(row[1]) if row else None + + if current_version: + txn.execute( + database_engine.convert_param_style( + "SELECT file FROM applied_schema_deltas WHERE version >= ?" + ), + (current_version,) + ) + applied_deltas = [d for d, in txn.fetchall()] + return current_version, applied_deltas, upgraded + + return None + + +def prepare_sqlite3_database(db_conn): + """This function should be called before `prepare_database` on sqlite3 + databases. + + Since we changed the way we store the current schema version and handle + updates to schemas, we need a way to upgrade from the old method to the + new. This only affects sqlite databases since they were the only ones + supported at the time. + """ + with db_conn: + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + create_schema = read_schema(schema_path) + db_conn.executescript(create_schema) + + c = db_conn.execute("SELECT * FROM schema_version") + rows = c.fetchall() + c.close() + + if not rows: + c = db_conn.execute("PRAGMA user_version") + row = c.fetchone() + c.close() + + if row and row[0]: + db_conn.execute( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)", + (row[0], False) + ) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 4a855ffd56..949396044e 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage import prepare_database +from synapse.storage._schema_prepare import prepare_database from ._base import IncorrectDatabaseSetup diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index d18e2808d1..a66815ef2d 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage import prepare_database, prepare_sqlite3_database +from synapse.storage._schema_prepare import ( + prepare_database, prepare_sqlite3_database +) class Sqlite3Engine(object): -- cgit 1.5.1 From 54414221e4ced47e632144afa7d768a7e252214c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 13 Oct 2015 11:43:12 +0100 Subject: Include invites in incremental sync --- synapse/handlers/sync.py | 31 +++++++++++++++++++++++++------ synapse/rest/client/v2_alpha/sync.py | 2 +- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 574412d6b5..d9e55d8a58 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -163,7 +163,7 @@ class SyncHandler(BaseHandler): invited = [] for event in room_list: if event.membership == Membership.JOIN: - room_sync = yield self.initial_sync_for_room( + room_sync = yield self.initial_sync_for_joined_room( event.room_id, sync_config, now_token, ) joined.append(room_sync) @@ -240,9 +240,9 @@ class SyncHandler(BaseHandler): ) if app_service: rooms = yield self.store.get_app_service_rooms(app_service) - room_ids = set(r.room_id for r in rooms) + joined_room_ids = set(r.room_id for r in rooms) else: - room_ids = yield rm_handler.get_joined_rooms_for_user( + joined_room_ids = yield rm_handler.get_joined_rooms_for_user( sync_config.user ) @@ -260,11 +260,17 @@ class SyncHandler(BaseHandler): if len(room_events) <= timeline_limit: # There is no gap in any of the rooms. Therefore we can just # partition the new events by room and return them. + invite_events = [] events_by_room_id = {} for event in room_events: events_by_room_id.setdefault(event.room_id, []).append(event) + if event.room_id not in joined_room_ids: + if (event.type == EventTypes.Member + and event.membership == Membership.INVITE + and event.state_key == sync_config.user.to_string()): + invite_events.append(event) - for room_id in room_ids: + for room_id in joined_room_ids: recents = events_by_room_id.get(room_id, []) state = [event for event in recents if event.is_state()] if recents: @@ -291,7 +297,15 @@ class SyncHandler(BaseHandler): if room_sync: joined.append(room_sync) else: - for room_id in room_ids: + invites = yield self.store.get_rooms_for_user_where_membership_is( + user_id=sync_config.user.to_string(), + membership_list=[Membership.INVITE], + ) + invite_events = yield self.store.get_events( + [invite.event_id for invite in invites] + ) + + for room_id in joined_room_ids: room_sync = yield self.incremental_sync_with_gap_for_room( room_id, sync_config, since_token, now_token, typing_by_room @@ -299,10 +313,15 @@ class SyncHandler(BaseHandler): if room_sync: joined.append(room_sync) + invited = [ + InvitedSyncResult(room_id=event.room_id, invite=event) + for event in invite_events + ] + defer.returnValue(SyncResult( presence=presence, joined=joined, - invited=[], + invited=invited, next_batch=now_token, )) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 399df9e772..fffecb24f5 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -177,7 +177,7 @@ class SyncRestServlet(RestServlet): invited_state = invite.get("unsigned", {}).pop("invite_room_state", []) invited_state.append(invite) invited[room.room_id] = { - "invite_state": { "events": invited_state } + "invite_state": {"events": invited_state} } return invited -- cgit 1.5.1 From ec398af41c4d276abb02279efbcbb0aa08a4cbc8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Oct 2015 11:41:04 +0100 Subject: Expose error more nicely --- synapse/app/homeserver.py | 5 +- synapse/storage/__init__.py | 3 - synapse/storage/_schema_prepare.py | 395 ------------------------------------ synapse/storage/engines/postgres.py | 2 +- synapse/storage/engines/sqlite3.py | 2 +- synapse/storage/schema_prepare.py | 395 ++++++++++++++++++++++++++++++++++++ tests/utils.py | 2 +- 7 files changed, 400 insertions(+), 404 deletions(-) delete mode 100644 synapse/storage/_schema_prepare.py create mode 100644 synapse/storage/schema_prepare.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 190b03e2f7..b284d07cf0 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -35,9 +35,8 @@ if __name__ == '__main__': from synapse.storage.engines import create_engine, IncorrectDatabaseSetup -from synapse.storage import ( - are_all_users_on_domain, UpgradeDatabaseException, -) +from synapse.storage import are_all_users_on_domain +from synapse.storage.schema_prepare import UpgradeDatabaseException from synapse.server import HomeServer diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4be629bff8..48a0633746 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -41,9 +41,6 @@ from .end_to_end_keys import EndToEndKeyStore from .receipts import ReceiptsStore -from ._schema_prepare import UpgradeDatabaseException - -__all__ = [UpgradeDatabaseException] import logging diff --git a/synapse/storage/_schema_prepare.py b/synapse/storage/_schema_prepare.py deleted file mode 100644 index 1ddf55be4d..0000000000 --- a/synapse/storage/_schema_prepare.py +++ /dev/null @@ -1,395 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import fnmatch -import imp -import logging -import os -import re - - -logger = logging.getLogger(__name__) - - -# Remember to update this number every time a change is made to database -# schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 24 - -dir_path = os.path.abspath(os.path.dirname(__file__)) - - -def read_schema(path): - """ Read the named database schema. - - Args: - path: Path of the database schema. - Returns: - A string containing the database schema. - """ - with open(path) as schema_file: - return schema_file.read() - - -class PrepareDatabaseException(Exception): - pass - - -class UpgradeDatabaseException(PrepareDatabaseException): - pass - - -def prepare_database(db_conn, database_engine): - """Prepares a database for usage. Will either create all necessary tables - or upgrade from an older schema version. - """ - try: - cur = db_conn.cursor() - version_info = _get_or_create_schema_state(cur, database_engine) - - if version_info: - user_version, delta_files, upgraded = version_info - _upgrade_existing_database( - cur, user_version, delta_files, upgraded, database_engine - ) - else: - _setup_new_database(cur, database_engine) - - # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) - - cur.close() - db_conn.commit() - except: - db_conn.rollback() - raise - - -def _setup_new_database(cur, database_engine): - """Sets up the database by finding a base set of "full schemas" and then - applying any necessary deltas. - - The "full_schemas" directory has subdirectories named after versions. This - function searches for the highest version less than or equal to - `SCHEMA_VERSION` and executes all .sql files in that directory. - - The function will then apply all deltas for all versions after the base - version. - - Example directory structure: - - schema/ - delta/ - ... - full_schemas/ - 3/ - test.sql - ... - 11/ - foo.sql - bar.sql - ... - - In the example foo.sql and bar.sql would be run, and then any delta files - for versions strictly greater than 11. - """ - current_dir = os.path.join(dir_path, "schema", "full_schemas") - directory_entries = os.listdir(current_dir) - - valid_dirs = [] - pattern = re.compile(r"^\d+(\.sql)?$") - for filename in directory_entries: - match = pattern.match(filename) - abs_path = os.path.join(current_dir, filename) - if match and os.path.isdir(abs_path): - ver = int(match.group(0)) - if ver <= SCHEMA_VERSION: - valid_dirs.append((ver, abs_path)) - else: - logger.warn("Unexpected entry in 'full_schemas': %s", filename) - - if not valid_dirs: - raise PrepareDatabaseException( - "Could not find a suitable base set of full schemas" - ) - - max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0]) - - logger.debug("Initialising schema v%d", max_current_ver) - - directory_entries = os.listdir(sql_dir) - - for filename in fnmatch.filter(directory_entries, "*.sql"): - sql_loc = os.path.join(sql_dir, filename) - logger.debug("Applying schema %s", sql_loc) - executescript(cur, sql_loc) - - cur.execute( - database_engine.convert_param_style( - "INSERT INTO schema_version (version, upgraded)" - " VALUES (?,?)" - ), - (max_current_ver, False,) - ) - - _upgrade_existing_database( - cur, - current_version=max_current_ver, - applied_delta_files=[], - upgraded=False, - database_engine=database_engine, - ) - - -def _upgrade_existing_database(cur, current_version, applied_delta_files, - upgraded, database_engine): - """Upgrades an existing database. - - Delta files can either be SQL stored in *.sql files, or python modules - in *.py. - - There can be multiple delta files per version. Synapse will keep track of - which delta files have been applied, and will apply any that haven't been - even if there has been no version bump. This is useful for development - where orthogonal schema changes may happen on separate branches. - - Different delta files for the same version *must* be orthogonal and give - the same result when applied in any order. No guarantees are made on the - order of execution of these scripts. - - This is a no-op of current_version == SCHEMA_VERSION. - - Example directory structure: - - schema/ - delta/ - 11/ - foo.sql - ... - 12/ - foo.sql - bar.py - ... - full_schemas/ - ... - - In the example, if current_version is 11, then foo.sql will be run if and - only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in - some arbitrary order. - - Args: - cur (Cursor) - current_version (int): The current version of the schema. - applied_delta_files (list): A list of deltas that have already been - applied. - upgraded (bool): Whether the current version was generated by having - applied deltas or from full schema file. If `True` the function - will never apply delta files for the given `current_version`, since - the current_version wasn't generated by applying those delta files. - """ - - if current_version > SCHEMA_VERSION: - raise ValueError( - "Cannot use this database as it is too " + - "new for the server to understand" - ) - - start_ver = current_version - if not upgraded: - start_ver += 1 - - logger.debug("applied_delta_files: %s", applied_delta_files) - - for v in range(start_ver, SCHEMA_VERSION + 1): - logger.debug("Upgrading schema to v%d", v) - - delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) - - try: - directory_entries = os.listdir(delta_dir) - except OSError: - logger.exception("Could not open delta dir for version %d", v) - raise UpgradeDatabaseException( - "Could not open delta dir for version %d" % (v,) - ) - - directory_entries.sort() - for file_name in directory_entries: - relative_path = os.path.join(str(v), file_name) - logger.debug("Found file: %s", relative_path) - if relative_path in applied_delta_files: - continue - - absolute_path = os.path.join( - dir_path, "schema", "delta", relative_path, - ) - root_name, ext = os.path.splitext(file_name) - if ext == ".py": - # This is a python upgrade module. We need to import into some - # package and then execute its `run_upgrade` function. - module_name = "synapse.storage.v%d_%s" % ( - v, root_name - ) - with open(absolute_path) as python_file: - module = imp.load_source( - module_name, absolute_path, python_file - ) - logger.debug("Running script %s", relative_path) - module.run_upgrade(cur, database_engine) - elif ext == ".pyc": - # Sometimes .pyc files turn up anyway even though we've - # disabled their generation; e.g. from distribution package - # installers. Silently skip it - pass - elif ext == ".sql": - # A plain old .sql file, just read and execute it - logger.debug("Applying schema %s", relative_path) - executescript(cur, absolute_path) - else: - # Not a valid delta file. - logger.warn( - "Found directory entry that did not end in .py or" - " .sql: %s", - relative_path, - ) - continue - - # Mark as done. - cur.execute( - database_engine.convert_param_style( - "INSERT INTO applied_schema_deltas (version, file)" - " VALUES (?,?)", - ), - (v, relative_path) - ) - - cur.execute("DELETE FROM schema_version") - cur.execute( - database_engine.convert_param_style( - "INSERT INTO schema_version (version, upgraded)" - " VALUES (?,?)", - ), - (v, True) - ) - - -def get_statements(f): - statement_buffer = "" - in_comment = False # If we're in a /* ... */ style comment - - for line in f: - line = line.strip() - - if in_comment: - # Check if this line contains an end to the comment - comments = line.split("*/", 1) - if len(comments) == 1: - continue - line = comments[1] - in_comment = False - - # Remove inline block comments - line = re.sub(r"/\*.*\*/", " ", line) - - # Does this line start a comment? - comments = line.split("/*", 1) - if len(comments) > 1: - line = comments[0] - in_comment = True - - # Deal with line comments - line = line.split("--", 1)[0] - line = line.split("//", 1)[0] - - # Find *all* semicolons. We need to treat first and last entry - # specially. - statements = line.split(";") - - # We must prepend statement_buffer to the first statement - first_statement = "%s %s" % ( - statement_buffer.strip(), - statements[0].strip() - ) - statements[0] = first_statement - - # Every entry, except the last, is a full statement - for statement in statements[:-1]: - yield statement.strip() - - # The last entry did *not* end in a semicolon, so we store it for the - # next semicolon we find - statement_buffer = statements[-1].strip() - - -def executescript(txn, schema_path): - with open(schema_path, 'r') as f: - for statement in get_statements(f): - txn.execute(statement) - - -def _get_or_create_schema_state(txn, database_engine): - # Bluntly try creating the schema_version tables. - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - executescript(txn, schema_path) - - txn.execute("SELECT version, upgraded FROM schema_version") - row = txn.fetchone() - current_version = int(row[0]) if row else None - upgraded = bool(row[1]) if row else None - - if current_version: - txn.execute( - database_engine.convert_param_style( - "SELECT file FROM applied_schema_deltas WHERE version >= ?" - ), - (current_version,) - ) - applied_deltas = [d for d, in txn.fetchall()] - return current_version, applied_deltas, upgraded - - return None - - -def prepare_sqlite3_database(db_conn): - """This function should be called before `prepare_database` on sqlite3 - databases. - - Since we changed the way we store the current schema version and handle - updates to schemas, we need a way to upgrade from the old method to the - new. This only affects sqlite databases since they were the only ones - supported at the time. - """ - with db_conn: - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - create_schema = read_schema(schema_path) - db_conn.executescript(create_schema) - - c = db_conn.execute("SELECT * FROM schema_version") - rows = c.fetchall() - c.close() - - if not rows: - c = db_conn.execute("PRAGMA user_version") - row = c.fetchone() - c.close() - - if row and row[0]: - db_conn.execute( - "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", - (row[0], False) - ) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 949396044e..7e45dabf4c 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage._schema_prepare import prepare_database +from synapse.storage.schema_prepare import prepare_database from ._base import IncorrectDatabaseSetup diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index a66815ef2d..0eeaa45d19 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage._schema_prepare import ( +from synapse.storage.schema_prepare import ( prepare_database, prepare_sqlite3_database ) diff --git a/synapse/storage/schema_prepare.py b/synapse/storage/schema_prepare.py new file mode 100644 index 0000000000..1ddf55be4d --- /dev/null +++ b/synapse/storage/schema_prepare.py @@ -0,0 +1,395 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import imp +import logging +import os +import re + + +logger = logging.getLogger(__name__) + + +# Remember to update this number every time a change is made to database +# schema files, so the users will be informed on server restarts. +SCHEMA_VERSION = 24 + +dir_path = os.path.abspath(os.path.dirname(__file__)) + + +def read_schema(path): + """ Read the named database schema. + + Args: + path: Path of the database schema. + Returns: + A string containing the database schema. + """ + with open(path) as schema_file: + return schema_file.read() + + +class PrepareDatabaseException(Exception): + pass + + +class UpgradeDatabaseException(PrepareDatabaseException): + pass + + +def prepare_database(db_conn, database_engine): + """Prepares a database for usage. Will either create all necessary tables + or upgrade from an older schema version. + """ + try: + cur = db_conn.cursor() + version_info = _get_or_create_schema_state(cur, database_engine) + + if version_info: + user_version, delta_files, upgraded = version_info + _upgrade_existing_database( + cur, user_version, delta_files, upgraded, database_engine + ) + else: + _setup_new_database(cur, database_engine) + + # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) + + cur.close() + db_conn.commit() + except: + db_conn.rollback() + raise + + +def _setup_new_database(cur, database_engine): + """Sets up the database by finding a base set of "full schemas" and then + applying any necessary deltas. + + The "full_schemas" directory has subdirectories named after versions. This + function searches for the highest version less than or equal to + `SCHEMA_VERSION` and executes all .sql files in that directory. + + The function will then apply all deltas for all versions after the base + version. + + Example directory structure: + + schema/ + delta/ + ... + full_schemas/ + 3/ + test.sql + ... + 11/ + foo.sql + bar.sql + ... + + In the example foo.sql and bar.sql would be run, and then any delta files + for versions strictly greater than 11. + """ + current_dir = os.path.join(dir_path, "schema", "full_schemas") + directory_entries = os.listdir(current_dir) + + valid_dirs = [] + pattern = re.compile(r"^\d+(\.sql)?$") + for filename in directory_entries: + match = pattern.match(filename) + abs_path = os.path.join(current_dir, filename) + if match and os.path.isdir(abs_path): + ver = int(match.group(0)) + if ver <= SCHEMA_VERSION: + valid_dirs.append((ver, abs_path)) + else: + logger.warn("Unexpected entry in 'full_schemas': %s", filename) + + if not valid_dirs: + raise PrepareDatabaseException( + "Could not find a suitable base set of full schemas" + ) + + max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0]) + + logger.debug("Initialising schema v%d", max_current_ver) + + directory_entries = os.listdir(sql_dir) + + for filename in fnmatch.filter(directory_entries, "*.sql"): + sql_loc = os.path.join(sql_dir, filename) + logger.debug("Applying schema %s", sql_loc) + executescript(cur, sql_loc) + + cur.execute( + database_engine.convert_param_style( + "INSERT INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), + (max_current_ver, False,) + ) + + _upgrade_existing_database( + cur, + current_version=max_current_ver, + applied_delta_files=[], + upgraded=False, + database_engine=database_engine, + ) + + +def _upgrade_existing_database(cur, current_version, applied_delta_files, + upgraded, database_engine): + """Upgrades an existing database. + + Delta files can either be SQL stored in *.sql files, or python modules + in *.py. + + There can be multiple delta files per version. Synapse will keep track of + which delta files have been applied, and will apply any that haven't been + even if there has been no version bump. This is useful for development + where orthogonal schema changes may happen on separate branches. + + Different delta files for the same version *must* be orthogonal and give + the same result when applied in any order. No guarantees are made on the + order of execution of these scripts. + + This is a no-op of current_version == SCHEMA_VERSION. + + Example directory structure: + + schema/ + delta/ + 11/ + foo.sql + ... + 12/ + foo.sql + bar.py + ... + full_schemas/ + ... + + In the example, if current_version is 11, then foo.sql will be run if and + only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in + some arbitrary order. + + Args: + cur (Cursor) + current_version (int): The current version of the schema. + applied_delta_files (list): A list of deltas that have already been + applied. + upgraded (bool): Whether the current version was generated by having + applied deltas or from full schema file. If `True` the function + will never apply delta files for the given `current_version`, since + the current_version wasn't generated by applying those delta files. + """ + + if current_version > SCHEMA_VERSION: + raise ValueError( + "Cannot use this database as it is too " + + "new for the server to understand" + ) + + start_ver = current_version + if not upgraded: + start_ver += 1 + + logger.debug("applied_delta_files: %s", applied_delta_files) + + for v in range(start_ver, SCHEMA_VERSION + 1): + logger.debug("Upgrading schema to v%d", v) + + delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) + + try: + directory_entries = os.listdir(delta_dir) + except OSError: + logger.exception("Could not open delta dir for version %d", v) + raise UpgradeDatabaseException( + "Could not open delta dir for version %d" % (v,) + ) + + directory_entries.sort() + for file_name in directory_entries: + relative_path = os.path.join(str(v), file_name) + logger.debug("Found file: %s", relative_path) + if relative_path in applied_delta_files: + continue + + absolute_path = os.path.join( + dir_path, "schema", "delta", relative_path, + ) + root_name, ext = os.path.splitext(file_name) + if ext == ".py": + # This is a python upgrade module. We need to import into some + # package and then execute its `run_upgrade` function. + module_name = "synapse.storage.v%d_%s" % ( + v, root_name + ) + with open(absolute_path) as python_file: + module = imp.load_source( + module_name, absolute_path, python_file + ) + logger.debug("Running script %s", relative_path) + module.run_upgrade(cur, database_engine) + elif ext == ".pyc": + # Sometimes .pyc files turn up anyway even though we've + # disabled their generation; e.g. from distribution package + # installers. Silently skip it + pass + elif ext == ".sql": + # A plain old .sql file, just read and execute it + logger.debug("Applying schema %s", relative_path) + executescript(cur, absolute_path) + else: + # Not a valid delta file. + logger.warn( + "Found directory entry that did not end in .py or" + " .sql: %s", + relative_path, + ) + continue + + # Mark as done. + cur.execute( + database_engine.convert_param_style( + "INSERT INTO applied_schema_deltas (version, file)" + " VALUES (?,?)", + ), + (v, relative_path) + ) + + cur.execute("DELETE FROM schema_version") + cur.execute( + database_engine.convert_param_style( + "INSERT INTO schema_version (version, upgraded)" + " VALUES (?,?)", + ), + (v, True) + ) + + +def get_statements(f): + statement_buffer = "" + in_comment = False # If we're in a /* ... */ style comment + + for line in f: + line = line.strip() + + if in_comment: + # Check if this line contains an end to the comment + comments = line.split("*/", 1) + if len(comments) == 1: + continue + line = comments[1] + in_comment = False + + # Remove inline block comments + line = re.sub(r"/\*.*\*/", " ", line) + + # Does this line start a comment? + comments = line.split("/*", 1) + if len(comments) > 1: + line = comments[0] + in_comment = True + + # Deal with line comments + line = line.split("--", 1)[0] + line = line.split("//", 1)[0] + + # Find *all* semicolons. We need to treat first and last entry + # specially. + statements = line.split(";") + + # We must prepend statement_buffer to the first statement + first_statement = "%s %s" % ( + statement_buffer.strip(), + statements[0].strip() + ) + statements[0] = first_statement + + # Every entry, except the last, is a full statement + for statement in statements[:-1]: + yield statement.strip() + + # The last entry did *not* end in a semicolon, so we store it for the + # next semicolon we find + statement_buffer = statements[-1].strip() + + +def executescript(txn, schema_path): + with open(schema_path, 'r') as f: + for statement in get_statements(f): + txn.execute(statement) + + +def _get_or_create_schema_state(txn, database_engine): + # Bluntly try creating the schema_version tables. + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + executescript(txn, schema_path) + + txn.execute("SELECT version, upgraded FROM schema_version") + row = txn.fetchone() + current_version = int(row[0]) if row else None + upgraded = bool(row[1]) if row else None + + if current_version: + txn.execute( + database_engine.convert_param_style( + "SELECT file FROM applied_schema_deltas WHERE version >= ?" + ), + (current_version,) + ) + applied_deltas = [d for d, in txn.fetchall()] + return current_version, applied_deltas, upgraded + + return None + + +def prepare_sqlite3_database(db_conn): + """This function should be called before `prepare_database` on sqlite3 + databases. + + Since we changed the way we store the current schema version and handle + updates to schemas, we need a way to upgrade from the old method to the + new. This only affects sqlite databases since they were the only ones + supported at the time. + """ + with db_conn: + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + create_schema = read_schema(schema_path) + db_conn.executescript(create_schema) + + c = db_conn.execute("SELECT * FROM schema_version") + rows = c.fetchall() + c.close() + + if not rows: + c = db_conn.execute("PRAGMA user_version") + row = c.fetchone() + c.close() + + if row and row[0]: + db_conn.execute( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)", + (row[0], False) + ) diff --git a/tests/utils.py b/tests/utils.py index dd19a16fc7..6eb575bd09 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -16,7 +16,7 @@ from synapse.http.server import HttpServer from synapse.api.errors import cs_error, CodeMessageException, StoreError from synapse.api.constants import EventTypes -from synapse.storage import prepare_database +from synapse.storage.schema_prepare import prepare_database from synapse.storage.engines import create_engine from synapse.server import HomeServer -- cgit 1.5.1 From cfd39d6b55fad5b176f1883e1bc87ed8e14acf42 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Oct 2015 13:47:50 +0100 Subject: Add SQLite support --- synapse/storage/search.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 238df38440..5843f80876 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -17,6 +17,7 @@ from twisted.internet import defer from _base import SQLBaseStore from synapse.api.constants import KnownRoomEventKeys, SearchConstraintTypes +from synapse.storage.engines import PostgresEngine class SearchStore(SQLBaseStore): @@ -48,11 +49,17 @@ class SearchStore(SQLBaseStore): "(%s)" % (" OR ".join(local_clauses),) ) - sql = ( - "SELECT ts_rank_cd(vector, query) AS rank, event_id" - " FROM plainto_tsquery('english', ?) as query, event_search" - " WHERE vector @@ query" - ) + if isinstance(self.database_engine, PostgresEngine): + sql = ( + "SELECT ts_rank_cd(vector, query) AS rank, event_id" + " FROM plainto_tsquery('english', ?) as query, event_search" + " WHERE vector @@ query" + ) + else: + sql = ( + "SELECT 0 as rank, event_id FROM event_search" + " WHERE value MATCH ?" + ) for clause in clauses: sql += " AND " + clause -- cgit 1.5.1 From 17c80c8a3d92acca5bda9b0fc7d9898547476563 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Oct 2015 13:56:22 +0100 Subject: rename schema_prepare to prepare_database --- synapse/app/homeserver.py | 2 +- synapse/storage/engines/postgres.py | 2 +- synapse/storage/engines/sqlite3.py | 2 +- synapse/storage/prepare_database.py | 395 ++++++++++++++++++++++++++++++++++++ synapse/storage/schema_prepare.py | 395 ------------------------------------ tests/utils.py | 2 +- 6 files changed, 399 insertions(+), 399 deletions(-) create mode 100644 synapse/storage/prepare_database.py delete mode 100644 synapse/storage/schema_prepare.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b284d07cf0..af53acb369 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -36,7 +36,7 @@ if __name__ == '__main__': from synapse.storage.engines import create_engine, IncorrectDatabaseSetup from synapse.storage import are_all_users_on_domain -from synapse.storage.schema_prepare import UpgradeDatabaseException +from synapse.storage.prepare_database import UpgradeDatabaseException from synapse.server import HomeServer diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 7e45dabf4c..98d66e0a86 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage.schema_prepare import prepare_database +from synapse.storage.prepare_database import prepare_database from ._base import IncorrectDatabaseSetup diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 0eeaa45d19..bad3b5c5ac 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage.schema_prepare import ( +from synapse.storage.prepare_database import ( prepare_database, prepare_sqlite3_database ) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py new file mode 100644 index 0000000000..1ddf55be4d --- /dev/null +++ b/synapse/storage/prepare_database.py @@ -0,0 +1,395 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import imp +import logging +import os +import re + + +logger = logging.getLogger(__name__) + + +# Remember to update this number every time a change is made to database +# schema files, so the users will be informed on server restarts. +SCHEMA_VERSION = 24 + +dir_path = os.path.abspath(os.path.dirname(__file__)) + + +def read_schema(path): + """ Read the named database schema. + + Args: + path: Path of the database schema. + Returns: + A string containing the database schema. + """ + with open(path) as schema_file: + return schema_file.read() + + +class PrepareDatabaseException(Exception): + pass + + +class UpgradeDatabaseException(PrepareDatabaseException): + pass + + +def prepare_database(db_conn, database_engine): + """Prepares a database for usage. Will either create all necessary tables + or upgrade from an older schema version. + """ + try: + cur = db_conn.cursor() + version_info = _get_or_create_schema_state(cur, database_engine) + + if version_info: + user_version, delta_files, upgraded = version_info + _upgrade_existing_database( + cur, user_version, delta_files, upgraded, database_engine + ) + else: + _setup_new_database(cur, database_engine) + + # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) + + cur.close() + db_conn.commit() + except: + db_conn.rollback() + raise + + +def _setup_new_database(cur, database_engine): + """Sets up the database by finding a base set of "full schemas" and then + applying any necessary deltas. + + The "full_schemas" directory has subdirectories named after versions. This + function searches for the highest version less than or equal to + `SCHEMA_VERSION` and executes all .sql files in that directory. + + The function will then apply all deltas for all versions after the base + version. + + Example directory structure: + + schema/ + delta/ + ... + full_schemas/ + 3/ + test.sql + ... + 11/ + foo.sql + bar.sql + ... + + In the example foo.sql and bar.sql would be run, and then any delta files + for versions strictly greater than 11. + """ + current_dir = os.path.join(dir_path, "schema", "full_schemas") + directory_entries = os.listdir(current_dir) + + valid_dirs = [] + pattern = re.compile(r"^\d+(\.sql)?$") + for filename in directory_entries: + match = pattern.match(filename) + abs_path = os.path.join(current_dir, filename) + if match and os.path.isdir(abs_path): + ver = int(match.group(0)) + if ver <= SCHEMA_VERSION: + valid_dirs.append((ver, abs_path)) + else: + logger.warn("Unexpected entry in 'full_schemas': %s", filename) + + if not valid_dirs: + raise PrepareDatabaseException( + "Could not find a suitable base set of full schemas" + ) + + max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0]) + + logger.debug("Initialising schema v%d", max_current_ver) + + directory_entries = os.listdir(sql_dir) + + for filename in fnmatch.filter(directory_entries, "*.sql"): + sql_loc = os.path.join(sql_dir, filename) + logger.debug("Applying schema %s", sql_loc) + executescript(cur, sql_loc) + + cur.execute( + database_engine.convert_param_style( + "INSERT INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), + (max_current_ver, False,) + ) + + _upgrade_existing_database( + cur, + current_version=max_current_ver, + applied_delta_files=[], + upgraded=False, + database_engine=database_engine, + ) + + +def _upgrade_existing_database(cur, current_version, applied_delta_files, + upgraded, database_engine): + """Upgrades an existing database. + + Delta files can either be SQL stored in *.sql files, or python modules + in *.py. + + There can be multiple delta files per version. Synapse will keep track of + which delta files have been applied, and will apply any that haven't been + even if there has been no version bump. This is useful for development + where orthogonal schema changes may happen on separate branches. + + Different delta files for the same version *must* be orthogonal and give + the same result when applied in any order. No guarantees are made on the + order of execution of these scripts. + + This is a no-op of current_version == SCHEMA_VERSION. + + Example directory structure: + + schema/ + delta/ + 11/ + foo.sql + ... + 12/ + foo.sql + bar.py + ... + full_schemas/ + ... + + In the example, if current_version is 11, then foo.sql will be run if and + only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in + some arbitrary order. + + Args: + cur (Cursor) + current_version (int): The current version of the schema. + applied_delta_files (list): A list of deltas that have already been + applied. + upgraded (bool): Whether the current version was generated by having + applied deltas or from full schema file. If `True` the function + will never apply delta files for the given `current_version`, since + the current_version wasn't generated by applying those delta files. + """ + + if current_version > SCHEMA_VERSION: + raise ValueError( + "Cannot use this database as it is too " + + "new for the server to understand" + ) + + start_ver = current_version + if not upgraded: + start_ver += 1 + + logger.debug("applied_delta_files: %s", applied_delta_files) + + for v in range(start_ver, SCHEMA_VERSION + 1): + logger.debug("Upgrading schema to v%d", v) + + delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) + + try: + directory_entries = os.listdir(delta_dir) + except OSError: + logger.exception("Could not open delta dir for version %d", v) + raise UpgradeDatabaseException( + "Could not open delta dir for version %d" % (v,) + ) + + directory_entries.sort() + for file_name in directory_entries: + relative_path = os.path.join(str(v), file_name) + logger.debug("Found file: %s", relative_path) + if relative_path in applied_delta_files: + continue + + absolute_path = os.path.join( + dir_path, "schema", "delta", relative_path, + ) + root_name, ext = os.path.splitext(file_name) + if ext == ".py": + # This is a python upgrade module. We need to import into some + # package and then execute its `run_upgrade` function. + module_name = "synapse.storage.v%d_%s" % ( + v, root_name + ) + with open(absolute_path) as python_file: + module = imp.load_source( + module_name, absolute_path, python_file + ) + logger.debug("Running script %s", relative_path) + module.run_upgrade(cur, database_engine) + elif ext == ".pyc": + # Sometimes .pyc files turn up anyway even though we've + # disabled their generation; e.g. from distribution package + # installers. Silently skip it + pass + elif ext == ".sql": + # A plain old .sql file, just read and execute it + logger.debug("Applying schema %s", relative_path) + executescript(cur, absolute_path) + else: + # Not a valid delta file. + logger.warn( + "Found directory entry that did not end in .py or" + " .sql: %s", + relative_path, + ) + continue + + # Mark as done. + cur.execute( + database_engine.convert_param_style( + "INSERT INTO applied_schema_deltas (version, file)" + " VALUES (?,?)", + ), + (v, relative_path) + ) + + cur.execute("DELETE FROM schema_version") + cur.execute( + database_engine.convert_param_style( + "INSERT INTO schema_version (version, upgraded)" + " VALUES (?,?)", + ), + (v, True) + ) + + +def get_statements(f): + statement_buffer = "" + in_comment = False # If we're in a /* ... */ style comment + + for line in f: + line = line.strip() + + if in_comment: + # Check if this line contains an end to the comment + comments = line.split("*/", 1) + if len(comments) == 1: + continue + line = comments[1] + in_comment = False + + # Remove inline block comments + line = re.sub(r"/\*.*\*/", " ", line) + + # Does this line start a comment? + comments = line.split("/*", 1) + if len(comments) > 1: + line = comments[0] + in_comment = True + + # Deal with line comments + line = line.split("--", 1)[0] + line = line.split("//", 1)[0] + + # Find *all* semicolons. We need to treat first and last entry + # specially. + statements = line.split(";") + + # We must prepend statement_buffer to the first statement + first_statement = "%s %s" % ( + statement_buffer.strip(), + statements[0].strip() + ) + statements[0] = first_statement + + # Every entry, except the last, is a full statement + for statement in statements[:-1]: + yield statement.strip() + + # The last entry did *not* end in a semicolon, so we store it for the + # next semicolon we find + statement_buffer = statements[-1].strip() + + +def executescript(txn, schema_path): + with open(schema_path, 'r') as f: + for statement in get_statements(f): + txn.execute(statement) + + +def _get_or_create_schema_state(txn, database_engine): + # Bluntly try creating the schema_version tables. + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + executescript(txn, schema_path) + + txn.execute("SELECT version, upgraded FROM schema_version") + row = txn.fetchone() + current_version = int(row[0]) if row else None + upgraded = bool(row[1]) if row else None + + if current_version: + txn.execute( + database_engine.convert_param_style( + "SELECT file FROM applied_schema_deltas WHERE version >= ?" + ), + (current_version,) + ) + applied_deltas = [d for d, in txn.fetchall()] + return current_version, applied_deltas, upgraded + + return None + + +def prepare_sqlite3_database(db_conn): + """This function should be called before `prepare_database` on sqlite3 + databases. + + Since we changed the way we store the current schema version and handle + updates to schemas, we need a way to upgrade from the old method to the + new. This only affects sqlite databases since they were the only ones + supported at the time. + """ + with db_conn: + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + create_schema = read_schema(schema_path) + db_conn.executescript(create_schema) + + c = db_conn.execute("SELECT * FROM schema_version") + rows = c.fetchall() + c.close() + + if not rows: + c = db_conn.execute("PRAGMA user_version") + row = c.fetchone() + c.close() + + if row and row[0]: + db_conn.execute( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)", + (row[0], False) + ) diff --git a/synapse/storage/schema_prepare.py b/synapse/storage/schema_prepare.py deleted file mode 100644 index 1ddf55be4d..0000000000 --- a/synapse/storage/schema_prepare.py +++ /dev/null @@ -1,395 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import fnmatch -import imp -import logging -import os -import re - - -logger = logging.getLogger(__name__) - - -# Remember to update this number every time a change is made to database -# schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 24 - -dir_path = os.path.abspath(os.path.dirname(__file__)) - - -def read_schema(path): - """ Read the named database schema. - - Args: - path: Path of the database schema. - Returns: - A string containing the database schema. - """ - with open(path) as schema_file: - return schema_file.read() - - -class PrepareDatabaseException(Exception): - pass - - -class UpgradeDatabaseException(PrepareDatabaseException): - pass - - -def prepare_database(db_conn, database_engine): - """Prepares a database for usage. Will either create all necessary tables - or upgrade from an older schema version. - """ - try: - cur = db_conn.cursor() - version_info = _get_or_create_schema_state(cur, database_engine) - - if version_info: - user_version, delta_files, upgraded = version_info - _upgrade_existing_database( - cur, user_version, delta_files, upgraded, database_engine - ) - else: - _setup_new_database(cur, database_engine) - - # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) - - cur.close() - db_conn.commit() - except: - db_conn.rollback() - raise - - -def _setup_new_database(cur, database_engine): - """Sets up the database by finding a base set of "full schemas" and then - applying any necessary deltas. - - The "full_schemas" directory has subdirectories named after versions. This - function searches for the highest version less than or equal to - `SCHEMA_VERSION` and executes all .sql files in that directory. - - The function will then apply all deltas for all versions after the base - version. - - Example directory structure: - - schema/ - delta/ - ... - full_schemas/ - 3/ - test.sql - ... - 11/ - foo.sql - bar.sql - ... - - In the example foo.sql and bar.sql would be run, and then any delta files - for versions strictly greater than 11. - """ - current_dir = os.path.join(dir_path, "schema", "full_schemas") - directory_entries = os.listdir(current_dir) - - valid_dirs = [] - pattern = re.compile(r"^\d+(\.sql)?$") - for filename in directory_entries: - match = pattern.match(filename) - abs_path = os.path.join(current_dir, filename) - if match and os.path.isdir(abs_path): - ver = int(match.group(0)) - if ver <= SCHEMA_VERSION: - valid_dirs.append((ver, abs_path)) - else: - logger.warn("Unexpected entry in 'full_schemas': %s", filename) - - if not valid_dirs: - raise PrepareDatabaseException( - "Could not find a suitable base set of full schemas" - ) - - max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0]) - - logger.debug("Initialising schema v%d", max_current_ver) - - directory_entries = os.listdir(sql_dir) - - for filename in fnmatch.filter(directory_entries, "*.sql"): - sql_loc = os.path.join(sql_dir, filename) - logger.debug("Applying schema %s", sql_loc) - executescript(cur, sql_loc) - - cur.execute( - database_engine.convert_param_style( - "INSERT INTO schema_version (version, upgraded)" - " VALUES (?,?)" - ), - (max_current_ver, False,) - ) - - _upgrade_existing_database( - cur, - current_version=max_current_ver, - applied_delta_files=[], - upgraded=False, - database_engine=database_engine, - ) - - -def _upgrade_existing_database(cur, current_version, applied_delta_files, - upgraded, database_engine): - """Upgrades an existing database. - - Delta files can either be SQL stored in *.sql files, or python modules - in *.py. - - There can be multiple delta files per version. Synapse will keep track of - which delta files have been applied, and will apply any that haven't been - even if there has been no version bump. This is useful for development - where orthogonal schema changes may happen on separate branches. - - Different delta files for the same version *must* be orthogonal and give - the same result when applied in any order. No guarantees are made on the - order of execution of these scripts. - - This is a no-op of current_version == SCHEMA_VERSION. - - Example directory structure: - - schema/ - delta/ - 11/ - foo.sql - ... - 12/ - foo.sql - bar.py - ... - full_schemas/ - ... - - In the example, if current_version is 11, then foo.sql will be run if and - only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in - some arbitrary order. - - Args: - cur (Cursor) - current_version (int): The current version of the schema. - applied_delta_files (list): A list of deltas that have already been - applied. - upgraded (bool): Whether the current version was generated by having - applied deltas or from full schema file. If `True` the function - will never apply delta files for the given `current_version`, since - the current_version wasn't generated by applying those delta files. - """ - - if current_version > SCHEMA_VERSION: - raise ValueError( - "Cannot use this database as it is too " + - "new for the server to understand" - ) - - start_ver = current_version - if not upgraded: - start_ver += 1 - - logger.debug("applied_delta_files: %s", applied_delta_files) - - for v in range(start_ver, SCHEMA_VERSION + 1): - logger.debug("Upgrading schema to v%d", v) - - delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) - - try: - directory_entries = os.listdir(delta_dir) - except OSError: - logger.exception("Could not open delta dir for version %d", v) - raise UpgradeDatabaseException( - "Could not open delta dir for version %d" % (v,) - ) - - directory_entries.sort() - for file_name in directory_entries: - relative_path = os.path.join(str(v), file_name) - logger.debug("Found file: %s", relative_path) - if relative_path in applied_delta_files: - continue - - absolute_path = os.path.join( - dir_path, "schema", "delta", relative_path, - ) - root_name, ext = os.path.splitext(file_name) - if ext == ".py": - # This is a python upgrade module. We need to import into some - # package and then execute its `run_upgrade` function. - module_name = "synapse.storage.v%d_%s" % ( - v, root_name - ) - with open(absolute_path) as python_file: - module = imp.load_source( - module_name, absolute_path, python_file - ) - logger.debug("Running script %s", relative_path) - module.run_upgrade(cur, database_engine) - elif ext == ".pyc": - # Sometimes .pyc files turn up anyway even though we've - # disabled their generation; e.g. from distribution package - # installers. Silently skip it - pass - elif ext == ".sql": - # A plain old .sql file, just read and execute it - logger.debug("Applying schema %s", relative_path) - executescript(cur, absolute_path) - else: - # Not a valid delta file. - logger.warn( - "Found directory entry that did not end in .py or" - " .sql: %s", - relative_path, - ) - continue - - # Mark as done. - cur.execute( - database_engine.convert_param_style( - "INSERT INTO applied_schema_deltas (version, file)" - " VALUES (?,?)", - ), - (v, relative_path) - ) - - cur.execute("DELETE FROM schema_version") - cur.execute( - database_engine.convert_param_style( - "INSERT INTO schema_version (version, upgraded)" - " VALUES (?,?)", - ), - (v, True) - ) - - -def get_statements(f): - statement_buffer = "" - in_comment = False # If we're in a /* ... */ style comment - - for line in f: - line = line.strip() - - if in_comment: - # Check if this line contains an end to the comment - comments = line.split("*/", 1) - if len(comments) == 1: - continue - line = comments[1] - in_comment = False - - # Remove inline block comments - line = re.sub(r"/\*.*\*/", " ", line) - - # Does this line start a comment? - comments = line.split("/*", 1) - if len(comments) > 1: - line = comments[0] - in_comment = True - - # Deal with line comments - line = line.split("--", 1)[0] - line = line.split("//", 1)[0] - - # Find *all* semicolons. We need to treat first and last entry - # specially. - statements = line.split(";") - - # We must prepend statement_buffer to the first statement - first_statement = "%s %s" % ( - statement_buffer.strip(), - statements[0].strip() - ) - statements[0] = first_statement - - # Every entry, except the last, is a full statement - for statement in statements[:-1]: - yield statement.strip() - - # The last entry did *not* end in a semicolon, so we store it for the - # next semicolon we find - statement_buffer = statements[-1].strip() - - -def executescript(txn, schema_path): - with open(schema_path, 'r') as f: - for statement in get_statements(f): - txn.execute(statement) - - -def _get_or_create_schema_state(txn, database_engine): - # Bluntly try creating the schema_version tables. - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - executescript(txn, schema_path) - - txn.execute("SELECT version, upgraded FROM schema_version") - row = txn.fetchone() - current_version = int(row[0]) if row else None - upgraded = bool(row[1]) if row else None - - if current_version: - txn.execute( - database_engine.convert_param_style( - "SELECT file FROM applied_schema_deltas WHERE version >= ?" - ), - (current_version,) - ) - applied_deltas = [d for d, in txn.fetchall()] - return current_version, applied_deltas, upgraded - - return None - - -def prepare_sqlite3_database(db_conn): - """This function should be called before `prepare_database` on sqlite3 - databases. - - Since we changed the way we store the current schema version and handle - updates to schemas, we need a way to upgrade from the old method to the - new. This only affects sqlite databases since they were the only ones - supported at the time. - """ - with db_conn: - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - create_schema = read_schema(schema_path) - db_conn.executescript(create_schema) - - c = db_conn.execute("SELECT * FROM schema_version") - rows = c.fetchall() - c.close() - - if not rows: - c = db_conn.execute("PRAGMA user_version") - row = c.fetchone() - c.close() - - if row and row[0]: - db_conn.execute( - "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", - (row[0], False) - ) diff --git a/tests/utils.py b/tests/utils.py index 6eb575bd09..4da51291a4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -16,7 +16,7 @@ from synapse.http.server import HttpServer from synapse.api.errors import cs_error, CodeMessageException, StoreError from synapse.api.constants import EventTypes -from synapse.storage.schema_prepare import prepare_database +from synapse.storage.prepare_database import prepare_database from synapse.storage.engines import create_engine from synapse.server import HomeServer -- cgit 1.5.1 From cacf0688c691517dab55c3cff294b6bac7f0d6e3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 13 Oct 2015 14:08:38 +0100 Subject: Add a get_invites_for_user method to the storage to find out the rooms a user is invited to --- synapse/handlers/sync.py | 8 ++------ synapse/storage/roommember.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index d9e55d8a58..380798b7ab 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -297,12 +297,8 @@ class SyncHandler(BaseHandler): if room_sync: joined.append(room_sync) else: - invites = yield self.store.get_rooms_for_user_where_membership_is( - user_id=sync_config.user.to_string(), - membership_list=[Membership.INVITE], - ) - invite_events = yield self.store.get_events( - [invite.event_id for invite in invites] + invite_events = yield self.store.get_invites_for_user( + sync_config.user.to_string() ) for room_id in joined_room_ids: diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 8c40d9a8a6..dd98dcfda8 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -110,6 +110,20 @@ class RoomMemberStore(SQLBaseStore): membership=membership, ).addCallback(self._get_events) + def get_invites_for_user(self, user_id): + """ Get all the invite events for a user + Args: + user_id (str): The user ID. + Returns: + A deferred list of event objects. + """ + + return self.get_rooms_for_user_where_membership_is( + user_id, [Membership.INVITE] + ).addCallback(lambda invites: self._get_events([ + invites.event_id for invite in invites + ])) + def get_rooms_for_user_where_membership_is(self, user_id, membership_list): """ Get all the rooms for this user where the membership for this user matches one in the membership list. -- cgit 1.5.1 From 2fa9e23e04a9bc2c9a192309db59fa8aae495432 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 13 Oct 2015 14:12:43 +0100 Subject: Update the v2 filters to support filtering presence and remove support for public/private user data --- synapse/api/filtering.py | 62 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 4d570b74f8..e79e91e7eb 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -54,7 +54,7 @@ class Filtering(object): ] room_level_definitions = [ - "state", "events", "ephemeral" + "state", "timeline", "ephemeral" ] for key in top_level_definitions: @@ -135,17 +135,23 @@ class Filter(object): def __init__(self, filter_json): self.filter_json = filter_json - def filter_public_user_data(self, events): - return self._filter_on_key(events, ["public_user_data"]) + def timeline_limit(self): + return self.filter_json.get("room", {}).get("timeline", {}).get("limit", 10) - def filter_private_user_data(self, events): - return self._filter_on_key(events, ["private_user_data"]) + def presence_limit(self): + return self.filter_json.get("presence", {}).get("limit", 10) + + def ephemeral_limit(self): + return self.filter_json.get("room", {}).get("ephemeral", {}).get("limit", 10) + + def filter_presence(self, events): + return self._filter_on_key(events, ["presence"]) def filter_room_state(self, events): return self._filter_on_key(events, ["room", "state"]) - def filter_room_events(self, events): - return self._filter_on_key(events, ["room", "events"]) + def filter_room_timeline(self, events): + return self._filter_on_key(events, ["room", "timeline"]) def filter_room_ephemeral(self, events): return self._filter_on_key(events, ["room", "ephemeral"]) @@ -169,11 +175,34 @@ class Filter(object): return [e for e in events if self._passes_definition(definition, e)] def _passes_definition(self, definition, event): + """Check if the event passes the filter definition + Args: + definition(dict): The filter definition to check against + event(dict or Event): The event to check + Returns: + True if the event passes the filter in the definition + """ + if type(event) is dict: + room_id = event.get("room_id") + sender = event.get("sender") + event_type = event["type"] + else: + room_id = getattr(event, "room_id", None) + sender = getattr(event, "sender", None) + event_type = event.type + return self._event_passes_definition( + definition, room_id, sender, event_type + ) + + def _event_passes_definition(self, definition, room_id, sender, + event_type): """Check if the event passes through the given definition. Args: definition(dict): The definition to check against. - event(Event): The event to check. + room_id(str): The id of the room this event is in or None. + sender(str): The sender of the event + event_type(str): The type of the event. Returns: True if the event passes through the filter. """ @@ -185,8 +214,7 @@ class Filter(object): # and 'not_types' then it is treated as only being in 'not_types') # room checks - if hasattr(event, "room_id"): - room_id = event.room_id + if room_id is not None: allow_rooms = definition.get("rooms", None) reject_rooms = definition.get("not_rooms", None) if reject_rooms and room_id in reject_rooms: @@ -195,9 +223,7 @@ class Filter(object): return False # sender checks - if hasattr(event, "sender"): - # Should we be including event.state_key for some event types? - sender = event.sender + if sender is not None: allow_senders = definition.get("senders", None) reject_senders = definition.get("not_senders", None) if reject_senders and sender in reject_senders: @@ -208,12 +234,12 @@ class Filter(object): # type checks if "not_types" in definition: for def_type in definition["not_types"]: - if self._event_matches_type(event, def_type): + if self._event_matches_type(event_type, def_type): return False if "types" in definition: included = False for def_type in definition["types"]: - if self._event_matches_type(event, def_type): + if self._event_matches_type(event_type, def_type): included = True break if not included: @@ -221,9 +247,9 @@ class Filter(object): return True - def _event_matches_type(self, event, def_type): + def _event_matches_type(self, event_type, def_type): if def_type.endswith("*"): type_prefix = def_type[:-1] - return event.type.startswith(type_prefix) + return event_type.startswith(type_prefix) else: - return event.type == def_type + return event_type == def_type -- cgit 1.5.1 From 889778155811277585debda837c359a4ae471706 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 13 Oct 2015 14:13:51 +0100 Subject: update filtering tests --- tests/api/test_filtering.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 65b2f590c8..6942cdac51 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -345,9 +345,9 @@ class FilteringTestCase(unittest.TestCase): ) @defer.inlineCallbacks - def test_filter_public_user_data_match(self): + def test_filter_presence_match(self): user_filter_json = { - "public_user_data": { + "presence": { "types": ["m.*"] } } @@ -368,13 +368,13 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - results = user_filter.filter_public_user_data(events=events) + results = user_filter.filter_presence(events=events) self.assertEquals(events, results) @defer.inlineCallbacks - def test_filter_public_user_data_no_match(self): + def test_filter_presence_no_match(self): user_filter_json = { - "public_user_data": { + "presence": { "types": ["m.*"] } } @@ -395,7 +395,7 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - results = user_filter.filter_public_user_data(events=events) + results = user_filter.filter_presence(events=events) self.assertEquals([], results) @defer.inlineCallbacks -- cgit 1.5.1 From 3e2a1297b513dc1fadb288c74684f6651a88016d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Oct 2015 15:22:14 +0100 Subject: Remove constraints in preperation of using filters --- synapse/handlers/search.py | 61 ++++++++-------------------------------------- synapse/storage/search.py | 30 ++++++++--------------- 2 files changed, 20 insertions(+), 71 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index d5c395061c..8864a921fc 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -18,7 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.constants import ( - EventTypes, KnownRoomEventKeys, Membership, SearchConstraintTypes + EventTypes, Membership, ) from synapse.api.errors import SynapseError from synapse.events.utils import serialize_event @@ -29,45 +29,6 @@ import logging logger = logging.getLogger(__name__) -KEYS_TO_ALLOWED_CONSTRAINT_TYPES = { - KnownRoomEventKeys.CONTENT_BODY: [SearchConstraintTypes.FTS], - KnownRoomEventKeys.CONTENT_MSGTYPE: [SearchConstraintTypes.EXACT], - KnownRoomEventKeys.CONTENT_NAME: [ - SearchConstraintTypes.FTS, - SearchConstraintTypes.EXACT, - SearchConstraintTypes.SUBSTRING, - ], - KnownRoomEventKeys.CONTENT_TOPIC: [SearchConstraintTypes.FTS], - KnownRoomEventKeys.SENDER: [SearchConstraintTypes.EXACT], - KnownRoomEventKeys.ORIGIN_SERVER_TS: [SearchConstraintTypes.RANGE], - KnownRoomEventKeys.ROOM_ID: [SearchConstraintTypes.EXACT], -} - - -class RoomConstraint(object): - def __init__(self, search_type, keys, value): - self.search_type = search_type - self.keys = keys - self.value = value - - @classmethod - def from_dict(cls, d): - search_type = d["type"] - keys = d["keys"] - - for key in keys: - if key not in KEYS_TO_ALLOWED_CONSTRAINT_TYPES: - raise SynapseError(400, "Unrecognized key %r", key) - - if search_type not in KEYS_TO_ALLOWED_CONSTRAINT_TYPES[key]: - raise SynapseError( - 400, - "Disallowed constraint type %r for key %r", search_type, key - ) - - return cls(search_type, keys, d["value"]) - - class SearchHandler(BaseHandler): def __init__(self, hs): @@ -121,22 +82,20 @@ class SearchHandler(BaseHandler): @defer.inlineCallbacks def search(self, user, content): - constraint_dicts = content["search_categories"]["room_events"]["constraints"] - constraints = [RoomConstraint.from_dict(c)for c in constraint_dicts] - - fts = False - for c in constraints: - if c.search_type == SearchConstraintTypes.FTS: - if fts: - raise SynapseError(400, "Only one constraint can be FTS") - fts = True + try: + search_term = content["search_categories"]["room_events"]["search_term"] + keys = content["search_categories"]["room_events"]["keys"] + except KeyError: + raise SynapseError(400, "Invalid search query") rooms = yield self.store.get_rooms_for_user_where_membership_is( - user.to_string(), membership_list=[Membership.JOIN, Membership.LEAVE], + user.to_string(), + membership_list=[Membership.JOIN], + # membership_list=[Membership.JOIN, Membership.LEAVE, Membership.Ban], ) room_ids = set(r.room_id for r in rooms) - rank_map, event_map = yield self.store.search_msgs(room_ids, constraints) + rank_map, event_map = yield self.store.search_msgs(room_ids, search_term, keys) allowed_events = yield self._filter_events_for_client( user.to_string(), event_map.values() diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 5843f80876..7a30ce25eb 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -16,38 +16,28 @@ from twisted.internet import defer from _base import SQLBaseStore -from synapse.api.constants import KnownRoomEventKeys, SearchConstraintTypes from synapse.storage.engines import PostgresEngine class SearchStore(SQLBaseStore): @defer.inlineCallbacks - def search_msgs(self, room_ids, constraints): + def search_msgs(self, room_ids, search_term, keys): clauses = [] args = [] - fts = None clauses.append( "room_id IN (%s)" % (",".join(["?"] * len(room_ids)),) ) args.extend(room_ids) - for c in constraints: - local_clauses = [] - if c.search_type == SearchConstraintTypes.FTS: - fts = c.value - for key in c.keys: - local_clauses.append("key = ?") - args.append(key) - elif c.search_type == SearchConstraintTypes.EXACT: - for key in c.keys: - if key == KnownRoomEventKeys.ROOM_ID: - for value in c.value: - local_clauses.append("room_id = ?") - args.append(value) - clauses.append( - "(%s)" % (" OR ".join(local_clauses),) - ) + local_clauses = [] + for key in keys: + local_clauses.append("key = ?") + args.append(key) + + clauses.append( + "(%s)" % (" OR ".join(local_clauses),) + ) if isinstance(self.database_engine, PostgresEngine): sql = ( @@ -67,7 +57,7 @@ class SearchStore(SQLBaseStore): sql += " ORDER BY rank DESC" results = yield self._execute( - "search_msgs", self.cursor_to_dict, sql, *([fts] + args) + "search_msgs", self.cursor_to_dict, sql, *([search_term] + args) ) events = yield self._get_events([r["event_id"] for r in results]) -- cgit 1.5.1 From 17dffef5ec74d789f68096c95d29cdcad57ce5c7 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 13 Oct 2015 15:48:12 +0100 Subject: Move event contents into third_party_layout field --- synapse/api/auth.py | 21 ++++++++++++--------- synapse/federation/federation_client.py | 4 ++-- synapse/handlers/_base.py | 2 +- synapse/handlers/federation.py | 5 +++-- synapse/handlers/room.py | 11 +++++++---- synapse/rest/client/v1/room.py | 3 ++- synapse/util/thirdpartyinvites.py | 10 ++++++++-- 7 files changed, 35 insertions(+), 21 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index adb9a776e0..ca280707c5 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -374,24 +374,24 @@ class Auth(object): return True def _verify_third_party_invite(self, event, auth_events): - for key in ThirdPartyInvites.JOIN_KEYS: - if key not in event.content: - return False - token = event.content["token"] + if not ThirdPartyInvites.join_has_third_party_invite(event.content): + return False + join_third_party_invite = event.content["third_party_invite"] + token = join_third_party_invite["token"] invite_event = auth_events.get( (EventTypes.ThirdPartyInvite, token,) ) if not invite_event: return False try: - public_key = event.content["public_key"] - key_validity_url = event.content["key_validity_url"] + public_key = join_third_party_invite["public_key"] + key_validity_url = join_third_party_invite["key_validity_url"] if invite_event.content["public_key"] != public_key: return False if invite_event.content["key_validity_url"] != key_validity_url: return False verify_key = nacl.signing.VerifyKey(decode_base64(public_key)) - encoded_signature = event.content["signature"] + encoded_signature = join_third_party_invite["signature"] signature = decode_base64(encoded_signature) verify_key.verify(token, signature) return True @@ -677,8 +677,11 @@ class Auth(object): if e_type == Membership.JOIN: if member_event and not is_public: auth_ids.append(member_event.event_id) - if ThirdPartyInvites.has_join_keys(event.content): - key = (EventTypes.ThirdPartyInvite, event.content["token"]) + if ThirdPartyInvites.join_has_third_party_invite(event.content): + key = ( + EventTypes.ThirdPartyInvite, + event.content["third_party_invite"]["token"] + ) invite = current_state.get(key) if invite: auth_ids.append(invite.event_id) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 06b0c7adcf..6be83d82e7 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -363,8 +363,8 @@ class FederationClient(FederationBase): continue args = {} - if ThirdPartyInvites.has_join_keys(content): - ThirdPartyInvites.copy_join_keys(content, args) + if ThirdPartyInvites.join_has_third_party_invite(content): + ThirdPartyInvites.copy_join_keys(content["third_party_invite"], args) try: ret = yield self.transport_layer.make_join( destination, room_id, user_id, args diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index faf99f5bd3..4165c56bed 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -127,7 +127,7 @@ class BaseHandler(object): if ( event.type == EventTypes.Member and event.content["membership"] == Membership.JOIN and - ThirdPartyInvites.has_join_keys(event.content) + ThirdPartyInvites.join_has_third_party_invite(event.content) ): yield ThirdPartyInvites.check_key_valid( self.hs.get_simple_http_client(), diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8197d8b2d0..8606c0d285 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -705,7 +705,8 @@ class FederationHandler(BaseHandler): """ event_content = {"membership": Membership.JOIN} if ThirdPartyInvites.has_join_keys(query): - ThirdPartyInvites.copy_join_keys(query, event_content) + event_content["third_party_invite"] = {} + ThirdPartyInvites.copy_join_keys(query, event_content["third_party_invite"]) builder = self.event_builder_factory.new({ "type": EventTypes.Member, @@ -721,7 +722,7 @@ class FederationHandler(BaseHandler): self.auth.check(event, auth_events=context.current_state) - if ThirdPartyInvites.has_join_keys(event.content): + if ThirdPartyInvites.join_has_third_party_invite(event.content): ThirdPartyInvites.check_key_valid(self.hs.get_simple_http_client(), event) defer.returnValue(event) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index b856b424a7..e07472b4b9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -483,10 +483,13 @@ class RoomMemberHandler(BaseHandler): should_do_dance = not self.hs.is_mine(inviter) room_hosts = [inviter.domain] - elif "sender" in event.content: - inviter = UserID.from_string(event.content["sender"]) - should_do_dance = not self.hs.is_mine(inviter) - room_hosts = [inviter.domain] + elif "third_party_invite" in event.content: + if "sender" in event.content["third_party_invite"]: + inviter = UserID.from_string( + event.content["third_party_invite"]["sender"] + ) + should_do_dance = not self.hs.is_mine(inviter) + room_hosts = [inviter.domain] else: # return the same error as join_room_alias does raise SynapseError(404, "No known servers") diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index ff84affea3..1cb6ba4f1f 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -456,7 +456,8 @@ class RoomMembershipRestServlet(ClientV1RestServlet): } if membership_action == "join" and ThirdPartyInvites.has_join_keys(content): - ThirdPartyInvites.copy_join_keys(content, event_content) + event_content["third_party_invite"] = {} + ThirdPartyInvites.copy_join_keys(content, event_content["third_party_invite"]) yield msg_handler.create_and_send_event( { diff --git a/synapse/util/thirdpartyinvites.py b/synapse/util/thirdpartyinvites.py index c30279de67..ad0f4e88e9 100644 --- a/synapse/util/thirdpartyinvites.py +++ b/synapse/util/thirdpartyinvites.py @@ -42,6 +42,12 @@ class ThirdPartyInvites(object): return False return True + @classmethod + def join_has_third_party_invite(cls, content): + if "third_party_invite" not in content: + return False + return cls.has_join_keys(content["third_party_invite"]) + @classmethod def copy_join_keys(cls, src, dst): for key in cls.JOIN_KEYS: @@ -53,8 +59,8 @@ class ThirdPartyInvites(object): def check_key_valid(cls, http_client, event): try: response = yield http_client.get_json( - event.content["key_validity_url"], - {"public_key": event.content["public_key"]} + event.content["third_party_invite"]["key_validity_url"], + {"public_key": event.content["third_party_invite"]["public_key"]} ) if not response["valid"]: raise AuthError(403, "Third party certificate was invalid") -- cgit 1.5.1 From 7ecd11accb68cc0f20e7ab84673df38413ba7cf7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Oct 2015 15:50:56 +0100 Subject: Add paranoia limit --- synapse/storage/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 7a30ce25eb..1b987161e2 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -54,7 +54,7 @@ class SearchStore(SQLBaseStore): for clause in clauses: sql += " AND " + clause - sql += " ORDER BY rank DESC" + sql += " ORDER BY rank DESC LIMIT 500" results = yield self._execute( "search_msgs", self.cursor_to_dict, sql, *([search_term] + args) -- cgit 1.5.1 From 7639c3d9e53cdb6222df6a8e1b12bc2a40612367 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 13 Oct 2015 17:13:04 +0100 Subject: Bounce all deferreds through the reactor to make debugging easier. If all deferreds wait a reactor tick before resolving then there is always a chance to add an errback to the deferred so that stacktraces get reported, rather than being discarded. --- synapse/app/homeserver.py | 2 ++ synapse/util/debug.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 synapse/util/debug.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index af53acb369..1c84242aa3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -33,6 +33,8 @@ if __name__ == '__main__': sys.stderr.writelines(message) sys.exit(1) + from synapse.util.debug import debug_deferreds + debug_deferreds() from synapse.storage.engines import create_engine, IncorrectDatabaseSetup from synapse.storage import are_all_users_on_domain diff --git a/synapse/util/debug.py b/synapse/util/debug.py new file mode 100644 index 0000000000..66ac12c291 --- /dev/null +++ b/synapse/util/debug.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer, reactor +from functools import wraps +from synapse.util.logcontext import LoggingContext, PreserveLoggingContext + +def with_logging_context(fn): + context = LoggingContext.current_context() + def restore_context_callback(x): + with PreserveLoggingContext(): + LoggingContext.thread_local.current_context = context + return fn(x) + return restore_context_callback + +def debug_deferreds(): + """Cause all deferreds to wait for a reactor tick before running their + callbacks. This increases the chance of getting a stack trace out of + a defer.inlineCallback since the code waiting on the deferred will get + a chance to add an errback before the deferred runs.""" + + # We are going to modify the __init__ method of defer.Deferred so we + # need to get a copy of the old method so we can still call it. + old__init__ = defer.Deferred.__init__ + + # We need to create a deferred to bounce the callbacks through the reactor + # but we don't want to add a callback when we create that deferred so we + # we create a new type of deferred that uses the old __init__ method. + # This is safe as long as the old __init__ method doesn't invoke an + # __init__ using super. + class Bouncer(defer.Deferred): + __init__ = old__init__ + + # We'll add this as a callback to all Deferreds. Twisted will wait until + # the bouncer deferred resolves before calling the callbacks of the + # original deferred. + def bounce_callback(x): + bouncer = Bouncer() + reactor.callLater(0, with_logging_context(bouncer.callback), x) + return bouncer + + # We'll add this as an errback to all Deferreds. Twisted will wait until + # the bouncer deferred resolves before calling the errbacks of the + # original deferred. + def bounce_errback(x): + bouncer = Bouncer() + reactor.callLater(0, with_logging_context(bouncer.errback), x) + return bouncer + + @wraps(old__init__) + def new__init__(self, *args, **kargs): + old__init__(self, *args, **kargs) + self.addCallbacks(bounce_callback, bounce_errback) + + defer.Deferred.__init__ = new__init__ + -- cgit 1.5.1 From 95e53ac53576de2980e8f7683b59f4fa7fce038b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 13 Oct 2015 17:18:24 +0100 Subject: Add some docstring --- synapse/api/auth.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index ca280707c5..c0762df567 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -374,6 +374,21 @@ class Auth(object): return True def _verify_third_party_invite(self, event, auth_events): + """ + Validates that the join event is authorized by a previous third-party invite. + + Checks that the public key, and keyserver, match those in the invite, + and that the join event has a signature issued using that public key. + + Args: + event: The m.room.member join event being validated. + auth_events: All relevant previous context events which may be used + for authorization decisions. + + Return: + True if the event fulfills the expectations of a previous third party + invite event. + """ if not ThirdPartyInvites.join_has_third_party_invite(event.content): return False join_third_party_invite = event.content["third_party_invite"] -- cgit 1.5.1 From 32d66738b0229aa7f011d203d0cb7963f950bb95 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 13 Oct 2015 17:18:29 +0100 Subject: Fix pep8 warnings. --- synapse/util/debug.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/synapse/util/debug.py b/synapse/util/debug.py index 66ac12c291..f6a5a841a4 100644 --- a/synapse/util/debug.py +++ b/synapse/util/debug.py @@ -17,13 +17,6 @@ from twisted.internet import defer, reactor from functools import wraps from synapse.util.logcontext import LoggingContext, PreserveLoggingContext -def with_logging_context(fn): - context = LoggingContext.current_context() - def restore_context_callback(x): - with PreserveLoggingContext(): - LoggingContext.thread_local.current_context = context - return fn(x) - return restore_context_callback def debug_deferreds(): """Cause all deferreds to wait for a reactor tick before running their @@ -31,6 +24,18 @@ def debug_deferreds(): a defer.inlineCallback since the code waiting on the deferred will get a chance to add an errback before the deferred runs.""" + # Helper method for retrieving and restoring the current logging context + # around a callback. + def with_logging_context(fn): + context = LoggingContext.current_context() + + def restore_context_callback(x): + with PreserveLoggingContext(): + LoggingContext.thread_local.current_context = context + return fn(x) + + return restore_context_callback + # We are going to modify the __init__ method of defer.Deferred so we # need to get a copy of the old method so we can still call it. old__init__ = defer.Deferred.__init__ @@ -65,4 +70,3 @@ def debug_deferreds(): self.addCallbacks(bounce_callback, bounce_errback) defer.Deferred.__init__ = new__init__ - -- cgit 1.5.1 From bb407cd62401686810c49616b8cf96aabefb30b4 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 13 Oct 2015 17:19:26 +0100 Subject: Re-add accidentally removed code --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8606c0d285..f10e5192e2 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -162,7 +162,7 @@ class FederationHandler(BaseHandler): auth_ids = [e_id for e_id, _ in e.auth_events] auth = { (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids + if e.event_id in auth_ids or e.type == EventTypes.Create } event_infos.append({ "event": e, -- cgit 1.5.1 From b68db6122263e7056a9805f5d0569d445b64fff8 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 13 Oct 2015 17:22:50 +0100 Subject: Add logging --- synapse/rest/client/v1/room.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 1cb6ba4f1f..2cb40df80b 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -498,8 +498,8 @@ class RoomMembershipRestServlet(ClientV1RestServlet): if "mxid" in data: # TODO: Validate the response signature and such defer.returnValue(data["mxid"]) - except IOError: - # TODO: Log something maybe? + except IOError as e: + logger.warn("Error from identity server lookup: %s" % (e,)) defer.returnValue(None) @defer.inlineCallbacks -- cgit 1.5.1 From 14edea1afff6e2b269f073d5561926348f9dee87 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 13 Oct 2015 17:47:58 +0100 Subject: Move logic into handler --- synapse/handlers/room.py | 132 ++++++++++++++++++++++++++++++++++++++++ synapse/rest/client/v1/room.py | 133 ++++++----------------------------------- 2 files changed, 150 insertions(+), 115 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index e07472b4b9..bbb457e823 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -547,6 +547,138 @@ class RoomMemberHandler(BaseHandler): suppress_auth=(not do_auth), ) + @defer.inlineCallbacks + def do_3pid_invite( + self, + room_id, + inviter, + medium, + address, + id_server, + display_name, + token_id, + txn_id + ): + invitee = yield self._lookup_3pid( + id_server, medium, address + ) + + if invitee: + # make sure it looks like a user ID; it'll throw if it's invalid. + UserID.from_string(invitee) + yield self.handlers.message_handler.create_and_send_event( + { + "type": EventTypes.Member, + "content": { + "membership": unicode("invite") + }, + "room_id": room_id, + "sender": inviter.to_string(), + "state_key": invitee, + }, + token_id=token_id, + txn_id=txn_id, + ) + else: + yield self._make_and_store_3pid_invite( + id_server, + display_name, + medium, + address, + room_id, + inviter, + token_id, + txn_id=txn_id + ) + + @defer.inlineCallbacks + def _lookup_3pid(self, id_server, medium, address): + """Looks up a 3pid in the passed identity server. + + Args: + id_server (str): The server name (including port, if required) + of the identity server to use. + medium (str): The type of the third party identifier (e.g. "email"). + address (str): The third party identifier (e.g. "foo@example.com"). + + Returns: + (str) the matrix ID of the 3pid, or None if it is not recognized. + """ + try: + data = yield self.hs.get_simple_http_client().get_json( + "https://%s/_matrix/identity/api/v1/lookup" % (id_server,), + { + "medium": medium, + "address": address, + } + ) + + if "mxid" in data: + # TODO: Validate the response signature and such + defer.returnValue(data["mxid"]) + except IOError as e: + logger.warn("Error from identity server lookup: %s" % (e,)) + defer.returnValue(None) + + @defer.inlineCallbacks + def _make_and_store_3pid_invite( + self, + id_server, + display_name, + medium, + address, + room_id, + user, + token_id, + txn_id + ): + token, public_key, key_validity_url = ( + yield self._ask_id_server_for_third_party_invite( + id_server, + medium, + address, + room_id, + user.to_string() + ) + ) + msg_handler = self.handlers.message_handler + yield msg_handler.create_and_send_event( + { + "type": EventTypes.ThirdPartyInvite, + "content": { + "display_name": display_name, + "key_validity_url": key_validity_url, + "public_key": public_key, + }, + "room_id": room_id, + "sender": user.to_string(), + "state_key": token, + }, + token_id=token_id, + txn_id=txn_id, + ) + + @defer.inlineCallbacks + def _ask_id_server_for_third_party_invite( + self, id_server, medium, address, room_id, sender): + is_url = "https://%s/_matrix/identity/api/v1/nonce-it-up" % (id_server,) + data = yield self.hs.get_simple_http_client().post_urlencoded_get_json( + is_url, + { + "medium": medium, + "address": address, + "room_id": room_id, + "sender": sender, + } + ) + # TODO: Check for success + token = data["token"] + public_key = data["public_key"] + key_validity_url = "https://%s/_matrix/identity/api/v1/pubkey/isvalid" % ( + id_server, + ) + defer.returnValue((token, public_key, key_validity_url)) + class RoomListHandler(BaseHandler): diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 2cb40df80b..1aca203744 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -414,34 +414,25 @@ class RoomMembershipRestServlet(ClientV1RestServlet): # target user is you unless it is an invite state_key = user.to_string() - if membership_action in ["invite", "ban", "kick"]: - try: + + if membership_action == "invite" and ThirdPartyInvites.has_invite_keys(content): + yield self.handlers.room_member_handler.do_3pid_invite( + room_id, + user, + content["medium"], + content["address"], + content["id_server"], + content["display_name"], + token_id, + txn_id + ) + defer.returnValue((200, {})) + return + elif membership_action in ["invite", "ban", "kick"]: + if "user_id" in content: state_key = content["user_id"] - except KeyError: - if ( - membership_action != "invite" or - not ThirdPartyInvites.has_invite_keys(content) - ): - raise SynapseError(400, "Missing user_id key.") - - id_server = content["id_server"] - medium = content["medium"] - address = content["address"] - display_name = content["display_name"] - state_key = yield self._lookup_3pid_user(id_server, medium, address) - if not state_key: - yield self._make_and_store_3pid_invite( - id_server, - display_name, - medium, - address, - room_id, - user, - token_id, - txn_id=txn_id - ) - defer.returnValue((200, {})) - return + else: + raise SynapseError(400, "Missing user_id key.") # make sure it looks like a user ID; it'll throw if it's invalid. UserID.from_string(state_key) @@ -473,94 +464,6 @@ class RoomMembershipRestServlet(ClientV1RestServlet): defer.returnValue((200, {})) - @defer.inlineCallbacks - def _lookup_3pid_user(self, id_server, medium, address): - """Looks up a 3pid in the passed identity server. - - Args: - id_server (str): The server name (including port, if required) - of the identity server to use. - medium (str): The type of the third party identifier (e.g. "email"). - address (str): The third party identifier (e.g. "foo@example.com"). - - Returns: - (str) the matrix ID of the 3pid, or None if it is not recognized. - """ - try: - data = yield self.hs.get_simple_http_client().get_json( - "https://%s/_matrix/identity/api/v1/lookup" % (id_server,), - { - "medium": medium, - "address": address, - } - ) - - if "mxid" in data: - # TODO: Validate the response signature and such - defer.returnValue(data["mxid"]) - except IOError as e: - logger.warn("Error from identity server lookup: %s" % (e,)) - defer.returnValue(None) - - @defer.inlineCallbacks - def _make_and_store_3pid_invite( - self, - id_server, - display_name, - medium, - address, - room_id, - user, - token_id, - txn_id - ): - token, public_key, key_validity_url = ( - yield self._ask_id_server_for_third_party_invite( - id_server, - medium, - address, - room_id, - user.to_string() - ) - ) - msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event( - { - "type": EventTypes.ThirdPartyInvite, - "content": { - "display_name": display_name, - "key_validity_url": key_validity_url, - "public_key": public_key, - }, - "room_id": room_id, - "sender": user.to_string(), - "state_key": token, - }, - token_id=token_id, - txn_id=txn_id, - ) - - @defer.inlineCallbacks - def _ask_id_server_for_third_party_invite( - self, id_server, medium, address, room_id, sender): - is_url = "https://%s/_matrix/identity/api/v1/nonce-it-up" % (id_server,) - data = yield self.hs.get_simple_http_client().post_urlencoded_get_json( - is_url, - { - "medium": medium, - "address": address, - "room_id": room_id, - "sender": sender, - } - ) - # TODO: Check for success - token = data["token"] - public_key = data["public_key"] - key_validity_url = "https://%s/_matrix/identity/api/v1/pubkey/isvalid" % ( - id_server, - ) - defer.returnValue((token, public_key, key_validity_url)) - @defer.inlineCallbacks def on_PUT(self, request, room_id, membership_action, txn_id): try: -- cgit 1.5.1 From 9020860479a9f70ae4d05ddcdc231d7e336474e3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 13 Oct 2015 17:50:44 +0100 Subject: Only turn on the twisted deferred debugging if full_twisted_stacktraces is set in the config --- synapse/app/homeserver.py | 3 --- synapse/config/logger.py | 8 ++++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 1c84242aa3..cf2fa221dc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -33,9 +33,6 @@ if __name__ == '__main__': sys.stderr.writelines(message) sys.exit(1) - from synapse.util.debug import debug_deferreds - debug_deferreds() - from synapse.storage.engines import create_engine, IncorrectDatabaseSetup from synapse.storage import are_all_users_on_domain from synapse.storage.prepare_database import UpgradeDatabaseException diff --git a/synapse/config/logger.py b/synapse/config/logger.py index bd0c17c861..a13dc170c4 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -22,6 +22,7 @@ import yaml from string import Template import os import signal +from synapse.util.debug import debug_deferreds DEFAULT_LOG_CONFIG = Template(""" @@ -69,6 +70,8 @@ class LoggingConfig(Config): self.verbosity = config.get("verbose", 0) self.log_config = self.abspath(config.get("log_config")) self.log_file = self.abspath(config.get("log_file")) + if config.get("full_twisted_stacktraces"): + debug_deferreds() def default_config(self, config_dir_path, server_name, **kwargs): log_file = self.abspath("homeserver.log") @@ -84,6 +87,11 @@ class LoggingConfig(Config): # A yaml python logging config file log_config: "%(log_config)s" + + # Stop twisted from discarding the stack traces of exceptions in + # deferreds by waiting a reactor tick before running a deferred's + # callbacks. + # full_twisted_stacktraces: true """ % locals() def read_arguments(self, args): -- cgit 1.5.1 From 1941eb315d692c44b0e21fb3fbf1b95eed138d53 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 13 Oct 2015 18:00:02 +0100 Subject: Enable stack traces for the demo scripts --- demo/start.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/demo/start.sh b/demo/start.sh index a90561488d..d90115ec97 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -38,6 +38,9 @@ for port in 8080 8081 8082; do perl -p -i -e 's/^enable_registration:.*/enable_registration: true/g' $DIR/etc/$port.config + echo "full_twisted_stacktraces: true" >> $DIR/etc/$port.config + echo "report_stats: false" >> $DIR/etc/$port.config + python -m synapse.app.homeserver \ --config-path "$DIR/etc/$port.config" \ -D \ -- cgit 1.5.1 From 0c38e8637ff549a21c763f02f52306b5c729d26b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 13 Oct 2015 18:00:38 +0100 Subject: Remove unnecessary class-wrapping --- synapse/api/auth.py | 6 +-- synapse/federation/federation_client.py | 6 +-- synapse/federation/federation_server.py | 6 +-- synapse/handlers/_base.py | 6 +-- synapse/handlers/federation.py | 13 ++++--- synapse/rest/client/v1/room.py | 11 +++--- synapse/util/third_party_invites.py | 69 +++++++++++++++++++++++++++++++++ synapse/util/thirdpartyinvites.py | 68 -------------------------------- 8 files changed, 94 insertions(+), 91 deletions(-) create mode 100644 synapse/util/third_party_invites.py delete mode 100644 synapse/util/thirdpartyinvites.py diff --git a/synapse/api/auth.py b/synapse/api/auth.py index c0762df567..e96d747b99 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -22,7 +22,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError from synapse.types import RoomID, UserID, EventID from synapse.util.logutils import log_function -from synapse.util.thirdpartyinvites import ThirdPartyInvites +from synapse.util import third_party_invites from unpaddedbase64 import decode_base64 import logging @@ -389,7 +389,7 @@ class Auth(object): True if the event fulfills the expectations of a previous third party invite event. """ - if not ThirdPartyInvites.join_has_third_party_invite(event.content): + if not third_party_invites.join_has_third_party_invite(event.content): return False join_third_party_invite = event.content["third_party_invite"] token = join_third_party_invite["token"] @@ -692,7 +692,7 @@ class Auth(object): if e_type == Membership.JOIN: if member_event and not is_public: auth_ids.append(member_event.event_id) - if ThirdPartyInvites.join_has_third_party_invite(event.content): + if third_party_invites.join_has_third_party_invite(event.content): key = ( EventTypes.ThirdPartyInvite, event.content["third_party_invite"]["token"] diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 6be83d82e7..d974e920c3 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -25,7 +25,7 @@ from synapse.api.errors import ( from synapse.util import unwrapFirstError from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.logutils import log_function -from synapse.util.thirdpartyinvites import ThirdPartyInvites +from synapse.util import third_party_invites from synapse.events import FrozenEvent import synapse.metrics @@ -363,8 +363,8 @@ class FederationClient(FederationBase): continue args = {} - if ThirdPartyInvites.join_has_third_party_invite(content): - ThirdPartyInvites.copy_join_keys(content["third_party_invite"], args) + if third_party_invites.join_has_third_party_invite(content): + args = third_party_invites.extract_join_keys(content) try: ret = yield self.transport_layer.make_join( destination, room_id, user_id, args diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index d71ab44271..7934f740e0 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -27,7 +27,7 @@ from synapse.api.errors import FederationError, SynapseError, Codes from synapse.crypto.event_signing import compute_event_signature -from synapse.util.thirdpartyinvites import ThirdPartyInvites +from synapse.util import third_party_invites import simplejson as json import logging @@ -232,8 +232,8 @@ class FederationServer(FederationBase): @defer.inlineCallbacks def on_make_join_request(self, room_id, user_id, query): threepid_details = {} - if ThirdPartyInvites.has_join_keys(query): - for k in ThirdPartyInvites.JOIN_KEYS: + if third_party_invites.has_join_keys(query): + for k in third_party_invites.JOIN_KEYS: if not isinstance(query[k], list) or len(query[k]) != 1: raise FederationError( "FATAL", diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 4165c56bed..97edec6ec6 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -21,7 +21,7 @@ from synapse.api.constants import Membership, EventTypes from synapse.types import UserID, RoomAlias from synapse.util.logcontext import PreserveLoggingContext -from synapse.util.thirdpartyinvites import ThirdPartyInvites +from synapse.util import third_party_invites import logging @@ -127,9 +127,9 @@ class BaseHandler(object): if ( event.type == EventTypes.Member and event.content["membership"] == Membership.JOIN and - ThirdPartyInvites.join_has_third_party_invite(event.content) + third_party_invites.join_has_third_party_invite(event.content) ): - yield ThirdPartyInvites.check_key_valid( + yield third_party_invites.check_key_valid( self.hs.get_simple_http_client(), event ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f10e5192e2..2b3c4cec8e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -39,7 +39,7 @@ from twisted.internet import defer import itertools import logging -from synapse.util.thirdpartyinvites import ThirdPartyInvites +from synapse.util import third_party_invites logger = logging.getLogger(__name__) @@ -704,9 +704,10 @@ class FederationHandler(BaseHandler): process it until the other server has signed it and sent it back. """ event_content = {"membership": Membership.JOIN} - if ThirdPartyInvites.has_join_keys(query): - event_content["third_party_invite"] = {} - ThirdPartyInvites.copy_join_keys(query, event_content["third_party_invite"]) + if third_party_invites.has_join_keys(query): + event_content["third_party_invite"] = ( + third_party_invites.extract_join_keys(query) + ) builder = self.event_builder_factory.new({ "type": EventTypes.Member, @@ -722,8 +723,8 @@ class FederationHandler(BaseHandler): self.auth.check(event, auth_events=context.current_state) - if ThirdPartyInvites.join_has_third_party_invite(event.content): - ThirdPartyInvites.check_key_valid(self.hs.get_simple_http_client(), event) + if third_party_invites.join_has_third_party_invite(event.content): + third_party_invites.check_key_valid(self.hs.get_simple_http_client(), event) defer.returnValue(event) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 1aca203744..1f45fcc6f1 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -26,7 +26,7 @@ from synapse.events.utils import serialize_event import simplejson as json import logging import urllib -from synapse.util.thirdpartyinvites import ThirdPartyInvites +from synapse.util import third_party_invites logger = logging.getLogger(__name__) @@ -415,7 +415,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): # target user is you unless it is an invite state_key = user.to_string() - if membership_action == "invite" and ThirdPartyInvites.has_invite_keys(content): + if membership_action == "invite" and third_party_invites.has_invite_keys(content): yield self.handlers.room_member_handler.do_3pid_invite( room_id, user, @@ -446,9 +446,10 @@ class RoomMembershipRestServlet(ClientV1RestServlet): "membership": unicode(membership_action), } - if membership_action == "join" and ThirdPartyInvites.has_join_keys(content): - event_content["third_party_invite"] = {} - ThirdPartyInvites.copy_join_keys(content, event_content["third_party_invite"]) + if membership_action == "join" and third_party_invites.has_join_keys(content): + event_content["third_party_invite"] = ( + third_party_invites.extract_join_keys(content) + ) yield msg_handler.create_and_send_event( { diff --git a/synapse/util/third_party_invites.py b/synapse/util/third_party_invites.py new file mode 100644 index 0000000000..b7e38c7ec3 --- /dev/null +++ b/synapse/util/third_party_invites.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer +from synapse.api.errors import AuthError + + +INVITE_KEYS = {"id_server", "medium", "address", "display_name"} + +JOIN_KEYS = { + "token", + "public_key", + "key_validity_url", + "signature", + "sender", +} + + +def has_invite_keys(content): + for key in INVITE_KEYS: + if key not in content: + return False + return True + + +def has_join_keys(content): + for key in JOIN_KEYS: + if key not in content: + return False + return True + + +def join_has_third_party_invite(content): + if "third_party_invite" not in content: + return False + return has_join_keys(content["third_party_invite"]) + + +def extract_join_keys(src): + return { + key: value + for key, value in src["third_party_invite"].items() + if key in JOIN_KEYS + } + + +@defer.inlineCallbacks +def check_key_valid(http_client, event): + try: + response = yield http_client.get_json( + event.content["third_party_invite"]["key_validity_url"], + {"public_key": event.content["third_party_invite"]["public_key"]} + ) + if not response["valid"]: + raise AuthError(403, "Third party certificate was invalid") + except IOError: + raise AuthError(403, "Third party certificate could not be checked") diff --git a/synapse/util/thirdpartyinvites.py b/synapse/util/thirdpartyinvites.py deleted file mode 100644 index ad0f4e88e9..0000000000 --- a/synapse/util/thirdpartyinvites.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer -from synapse.api.errors import AuthError - - -class ThirdPartyInvites(object): - INVITE_KEYS = {"id_server", "medium", "address", "display_name"} - - JOIN_KEYS = { - "token", - "public_key", - "key_validity_url", - "signature", - "sender", - } - - @classmethod - def has_invite_keys(cls, content): - for key in cls.INVITE_KEYS: - if key not in content: - return False - return True - - @classmethod - def has_join_keys(cls, content): - for key in cls.JOIN_KEYS: - if key not in content: - return False - return True - - @classmethod - def join_has_third_party_invite(cls, content): - if "third_party_invite" not in content: - return False - return cls.has_join_keys(content["third_party_invite"]) - - @classmethod - def copy_join_keys(cls, src, dst): - for key in cls.JOIN_KEYS: - if key in src: - dst[key] = src[key] - - @classmethod - @defer.inlineCallbacks - def check_key_valid(cls, http_client, event): - try: - response = yield http_client.get_json( - event.content["third_party_invite"]["key_validity_url"], - {"public_key": event.content["third_party_invite"]["public_key"]} - ) - if not response["valid"]: - raise AuthError(403, "Third party certificate was invalid") - except IOError: - raise AuthError(403, "Third party certificate could not be checked") -- cgit 1.5.1 From d82c5f7b5cb8b6dccbea87041844d3b156c74a15 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 13 Oct 2015 18:02:00 +0100 Subject: Use more descriptive error code --- synapse/util/third_party_invites.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/third_party_invites.py b/synapse/util/third_party_invites.py index b7e38c7ec3..45ae31d80c 100644 --- a/synapse/util/third_party_invites.py +++ b/synapse/util/third_party_invites.py @@ -66,4 +66,4 @@ def check_key_valid(http_client, event): if not response["valid"]: raise AuthError(403, "Third party certificate was invalid") except IOError: - raise AuthError(403, "Third party certificate could not be checked") + raise AuthError(502, "Third party certificate could not be checked") -- cgit 1.5.1 From 858634e1d0ca489bb546851d6ee052d548870b06 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 09:29:08 +0100 Subject: Remove unused room_id arg --- synapse/handlers/federation.py | 2 +- synapse/handlers/message.py | 10 +++++----- synapse/handlers/sync.py | 2 +- synapse/storage/state.py | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3882ba79ed..a710bdcfdb 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -242,7 +242,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _filter_events_for_server(self, server_name, room_id, events): event_to_state = yield self.store.get_state_for_events( - room_id, frozenset(e.event_id for e in events), + frozenset(e.event_id for e in events), types=( (EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, None), diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b70258697b..dfeeae76db 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -164,7 +164,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def _filter_events_for_client(self, user_id, room_id, events): event_id_to_state = yield self.store.get_state_for_events( - room_id, frozenset(e.event_id for e in events), + frozenset(e.event_id for e in events), types=( (EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, user_id), @@ -290,7 +290,7 @@ class MessageHandler(BaseHandler): elif member_event.membership == Membership.LEAVE: key = (event_type, state_key) room_state = yield self.store.get_state_for_events( - room_id, [member_event.event_id], [key] + [member_event.event_id], [key] ) data = room_state[member_event.event_id].get(key) @@ -314,7 +314,7 @@ class MessageHandler(BaseHandler): room_state = yield self.state_handler.get_current_state(room_id) elif member_event.membership == Membership.LEAVE: room_state = yield self.store.get_state_for_events( - room_id, [member_event.event_id], None + [member_event.event_id], None ) room_state = room_state[member_event.event_id] @@ -406,7 +406,7 @@ class MessageHandler(BaseHandler): elif event.membership == Membership.LEAVE: room_end_token = "s%d" % (event.stream_ordering,) deferred_room_state = self.store.get_state_for_events( - event.room_id, [event.event_id], None + [event.event_id], None ) deferred_room_state.addCallback( lambda states: states[event.event_id] @@ -499,7 +499,7 @@ class MessageHandler(BaseHandler): def _room_initial_sync_parted(self, user_id, room_id, pagin_config, member_event): room_state = yield self.store.get_state_for_events( - member_event.room_id, [member_event.event_id], None + [member_event.event_id], None ) room_state = room_state[member_event.event_id] diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9914ff6f9c..a8940de166 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -312,7 +312,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def _filter_events_for_client(self, user_id, room_id, events): event_id_to_state = yield self.store.get_state_for_events( - room_id, frozenset(e.event_id for e in events), + frozenset(e.event_id for e in events), types=( (EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, user_id), diff --git a/synapse/storage/state.py b/synapse/storage/state.py index e935b9443b..6f2a50d585 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -54,7 +54,7 @@ class StateStore(SQLBaseStore): defer.returnValue({}) event_to_groups = yield self._get_state_group_for_events( - room_id, event_ids, + event_ids, ) groups = set(event_to_groups.values()) @@ -208,7 +208,7 @@ class StateStore(SQLBaseStore): ) @defer.inlineCallbacks - def get_state_for_events(self, room_id, event_ids, types): + def get_state_for_events(self, event_ids, types): """Given a list of event_ids and type tuples, return a list of state dicts for each event. The state dicts will only have the type/state_keys that are in the `types` list. @@ -225,7 +225,7 @@ class StateStore(SQLBaseStore): The dicts are mappings from (type, state_key) -> state_events """ event_to_groups = yield self._get_state_group_for_events( - room_id, event_ids, + event_ids, ) groups = set(event_to_groups.values()) @@ -251,8 +251,8 @@ class StateStore(SQLBaseStore): ) @cachedList(cache=_get_state_group_for_event.cache, list_name="event_ids", - num_args=2) - def _get_state_group_for_events(self, room_id, event_ids): + num_args=1) + def _get_state_group_for_events(self, event_ids): """Returns mapping event_id -> state_group """ def f(txn): -- cgit 1.5.1 From d25b0f65ea9ab36dbf4285d86a1ca3e357f6ad1c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 09:46:31 +0100 Subject: Add TODO markers --- synapse/handlers/search.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 8864a921fc..79c1569868 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -88,6 +88,7 @@ class SearchHandler(BaseHandler): except KeyError: raise SynapseError(400, "Invalid search query") + # TODO: Search through left rooms too rooms = yield self.store.get_rooms_for_user_where_membership_is( user.to_string(), membership_list=[Membership.JOIN], @@ -95,6 +96,8 @@ class SearchHandler(BaseHandler): ) room_ids = set(r.room_id for r in rooms) + # TODO: Apply room filter to rooms list + rank_map, event_map = yield self.store.search_msgs(room_ids, search_term, keys) allowed_events = yield self._filter_events_for_client( @@ -111,7 +114,7 @@ class SearchHandler(BaseHandler): for e in allowed_events } - logger.info("returning: %r", results) + logger.info("Found %d results", len(results)) defer.returnValue({ "search_categories": { -- cgit 1.5.1 From 1d9e109820c1aec7193278b2b26042259329c144 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 09:49:00 +0100 Subject: More TODO markers --- synapse/handlers/search.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 79c1569868..8140c0b9d4 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -104,6 +104,9 @@ class SearchHandler(BaseHandler): user.to_string(), event_map.values() ) + # TODO: Filter allowed_events + # TODO: Add a limit + time_now = self.clock.time_msec() results = { -- cgit 1.5.1 From 99c7fbfef7729e6f3cceb9cea64f21d5a2c5b41f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 09:52:40 +0100 Subject: Fix to work with SQLite --- synapse/storage/room.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index e4e830944a..0527cee05d 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -19,6 +19,7 @@ from synapse.api.errors import StoreError from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks +from .engines import PostgresEngine import collections import logging @@ -202,10 +203,16 @@ class RoomStore(SQLBaseStore): ) def _store_event_search_txn(self, txn, event, key, value): - sql = ( - "INSERT INTO event_search (event_id, room_id, key, vector)" - " VALUES (?,?,?,to_tsvector('english', ?))" - ) + if isinstance(self.database_engine, PostgresEngine): + sql = ( + "INSERT INTO event_search (event_id, room_id, key, vector)" + " VALUES (?,?,?,to_tsvector('english', ?))" + ) + else: + sql = ( + "INSERT INTO event_search (event_id, room_id, key, value)" + " VALUES (?,?,?,?)" + ) txn.execute(sql, (event.event_id, event.room_id, key, value,)) -- cgit 1.5.1 From 8c9df8774e781da838efc18953785cfa1a2af0a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 10:35:50 +0100 Subject: Make 'keys' optional --- synapse/handlers/search.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 8140c0b9d4..7f1efe2b46 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -84,7 +84,9 @@ class SearchHandler(BaseHandler): def search(self, user, content): try: search_term = content["search_categories"]["room_events"]["search_term"] - keys = content["search_categories"]["room_events"]["keys"] + keys = content["search_categories"]["room_events"].get("keys", [ + "content.body", "content.name", "content.topic", + ]) except KeyError: raise SynapseError(400, "Invalid search query") -- cgit 1.5.1 From f45aaf0e35b447c15aace330d2daaa0005ad8461 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Oct 2015 10:36:55 +0100 Subject: Remove unused constatns --- synapse/api/constants.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 7c7f9ff957..008ee64727 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -84,22 +84,3 @@ class RoomCreationPreset(object): PRIVATE_CHAT = "private_chat" PUBLIC_CHAT = "public_chat" TRUSTED_PRIVATE_CHAT = "trusted_private_chat" - - -class SearchConstraintTypes(object): - FTS = "fts" - EXACT = "exact" - PREFIX = "prefix" - SUBSTRING = "substring" - RANGE = "range" - - -class KnownRoomEventKeys(object): - CONTENT_BODY = "content.body" - CONTENT_MSGTYPE = "content.msgtype" - CONTENT_NAME = "content.name" - CONTENT_TOPIC = "content.topic" - - SENDER = "sender" - ORIGIN_SERVER_TS = "origin_server_ts" - ROOM_ID = "room_id" -- cgit 1.5.1 From c185c1c4139f9ea1ad5b586a2cfd9f658cffcbb3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 14 Oct 2015 13:16:53 +0100 Subject: Fix v2 sync polling --- synapse/handlers/sync.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 380798b7ab..7b2d6e3457 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -82,7 +82,7 @@ class SyncResult(collections.namedtuple("SyncResult", [ events. """ return bool( - self.private_user_data or self.public_user_data or self.rooms + self.presence or self.joined or self.invited ) @@ -122,8 +122,8 @@ class SyncHandler(BaseHandler): ) result = yield self.notifier.wait_for_events( - sync_config.user, room_ids, - sync_config.filter, timeout, current_sync_callback + sync_config.user, room_ids, timeout, current_sync_callback, + from_token=since_token ) defer.returnValue(result) -- cgit 1.5.1 From 5338220d3a3926ea8fb659523a5087bb5b32a1ca Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 15 Oct 2015 10:39:19 +0100 Subject: synapse.util.emailutils was unused --- synapse/util/emailutils.py | 71 ---------------------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 synapse/util/emailutils.py diff --git a/synapse/util/emailutils.py b/synapse/util/emailutils.py deleted file mode 100644 index 7f9a77bf44..0000000000 --- a/synapse/util/emailutils.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" This module allows you to send out emails. -""" -import email.utils -import smtplib -import twisted.python.log -from email.mime.text import MIMEText -from email.mime.multipart import MIMEMultipart - -import logging - -logger = logging.getLogger(__name__) - - -class EmailException(Exception): - pass - - -def send_email(smtp_server, from_addr, to_addr, subject, body): - """Sends an email. - - Args: - smtp_server(str): The SMTP server to use. - from_addr(str): The address to send from. - to_addr(str): The address to send to. - subject(str): The subject of the email. - body(str): The plain text body of the email. - Raises: - EmailException if there was a problem sending the mail. - """ - if not smtp_server or not from_addr or not to_addr: - raise EmailException("Need SMTP server, from and to addresses. Check" - " the config to set these.") - - msg = MIMEMultipart('alternative') - msg['Subject'] = subject - msg['From'] = from_addr - msg['To'] = to_addr - plain_part = MIMEText(body) - msg.attach(plain_part) - - raw_from = email.utils.parseaddr(from_addr)[1] - raw_to = email.utils.parseaddr(to_addr)[1] - if not raw_from or not raw_to: - raise EmailException("Couldn't parse from/to address.") - - logger.info("Sending email to %s on server %s with subject %s", - to_addr, smtp_server, subject) - - try: - smtp = smtplib.SMTP(smtp_server) - smtp.sendmail(raw_from, raw_to, msg.as_string()) - smtp.quit() - except Exception as origException: - twisted.python.log.err() - ese = EmailException() - ese.cause = origException - raise ese -- cgit 1.5.1 From 1a934e8bfdf5fd8a2c89e6ada7b172a395e1a5f0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 15 Oct 2015 11:09:57 +0100 Subject: synapse.client.v1.login.LoginFallbackRestServlet and synapse.client.v1.login.PasswordResetRestServlet are unused --- synapse/rest/client/v1/login.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 2e3e4f39f3..dacc416055 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -192,36 +192,6 @@ class LoginRestServlet(ClientV1RestServlet): return (user, attributes) -class LoginFallbackRestServlet(ClientV1RestServlet): - PATTERN = client_path_pattern("/login/fallback$") - - def on_GET(self, request): - # TODO(kegan): This should be returning some HTML which is capable of - # hitting LoginRestServlet - return (200, {}) - - -class PasswordResetRestServlet(ClientV1RestServlet): - PATTERN = client_path_pattern("/login/reset") - - @defer.inlineCallbacks - def on_POST(self, request): - reset_info = _parse_json(request) - try: - email = reset_info["email"] - user_id = reset_info["user_id"] - handler = self.handlers.login_handler - yield handler.reset_password(user_id, email) - # purposefully give no feedback to avoid people hammering different - # combinations. - defer.returnValue((200, {})) - except KeyError: - raise SynapseError( - 400, - "Missing keys. Requires 'email' and 'user_id'." - ) - - class SAML2RestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login/saml2") -- cgit 1.5.1 From 643b5fcdc81d8f2e39a97dd1a94c78cd8769c613 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 15 Oct 2015 13:10:30 +0100 Subject: Look for keys on the right objects --- synapse/federation/federation_client.py | 4 +++- synapse/handlers/room.py | 4 ++-- synapse/util/third_party_invites.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index d974e920c3..f5b430e046 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -364,7 +364,9 @@ class FederationClient(FederationBase): args = {} if third_party_invites.join_has_third_party_invite(content): - args = third_party_invites.extract_join_keys(content) + args = third_party_invites.extract_join_keys( + content["third_party_invite"] + ) try: ret = yield self.transport_layer.make_join( destination, room_id, user_id, args diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index bbb457e823..6936cdc5fd 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -566,7 +566,7 @@ class RoomMemberHandler(BaseHandler): if invitee: # make sure it looks like a user ID; it'll throw if it's invalid. UserID.from_string(invitee) - yield self.handlers.message_handler.create_and_send_event( + yield self.hs.get_handlers().message_handler.create_and_send_event( { "type": EventTypes.Member, "content": { @@ -641,7 +641,7 @@ class RoomMemberHandler(BaseHandler): user.to_string() ) ) - msg_handler = self.handlers.message_handler + msg_handler = self.hs.get_handlers().message_handler yield msg_handler.create_and_send_event( { "type": EventTypes.ThirdPartyInvite, diff --git a/synapse/util/third_party_invites.py b/synapse/util/third_party_invites.py index 45ae31d80c..41e597d5b9 100644 --- a/synapse/util/third_party_invites.py +++ b/synapse/util/third_party_invites.py @@ -51,7 +51,7 @@ def join_has_third_party_invite(content): def extract_join_keys(src): return { key: value - for key, value in src["third_party_invite"].items() + for key, value in src.items() if key in JOIN_KEYS } -- cgit 1.5.1 From 6ffbcf45c6b8472c2cd6095a9dca2bd11a027b44 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 15 Oct 2015 13:12:52 +0100 Subject: Use non-placeholder name for endpoint --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6936cdc5fd..9ffa521aad 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -661,7 +661,7 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def _ask_id_server_for_third_party_invite( self, id_server, medium, address, room_id, sender): - is_url = "https://%s/_matrix/identity/api/v1/nonce-it-up" % (id_server,) + is_url = "https://%s/_matrix/identity/api/v1/store-invite" % (id_server,) data = yield self.hs.get_simple_http_client().post_urlencoded_get_json( is_url, { -- cgit 1.5.1 From 8189c4e3fdfcda3e4f449289363ac41fae521b8e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Oct 2015 15:06:13 +0100 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index d62294e6bb..e9ce0412ed 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.10.0-r2" +__version__ = "0.10.1-rc1" -- cgit 1.5.1 From e46cdc08cc9dc2c42b80be5ffb16fab4928308bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Oct 2015 15:16:18 +0100 Subject: Update change log --- CHANGES.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index f1d2c7a765..76060c77cb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,19 @@ +Changes in synapse v0.10.1-rc1 (2015-10-15) +=========================================== + +* Add CAS support, thanks to Steven Hammerton (PR #295, #296) +* Allow users to view the history of rooms that they have left. (PR #276, #294) +* Bundle in some room state in invites. (PR #275) +* Add flag on creation which disables federation of the room (PR #279) +* Atomically persist events when joining a room over federation (PR #283) +* Add support for ``m.room.canonical_alias`` (PR #287) +* Change default history visibility for private rooms (PR #271) +* Use Macaroons for ``access_token`` (PR #256, #229) +* Allow users to redact their own sent events (PR #262) +* Use tox for tests (PR #247) +* Split up syutil into separate libraries (PR #243) +* Add support for refresh tokens (PR #240) + Changes in synapse v0.10.0-r2 (2015-09-16) ========================================== -- cgit 1.5.1 From 6296590bf7fb4a8f43755d29ecb8a4819310e25d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 16 Oct 2015 10:50:32 +0100 Subject: Encode the filter JSON as UTF-8 before storing in the database. Because we are using a binary column type to store the filter JSON. --- synapse/storage/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index 8800116570..700da8d8c3 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -34,10 +34,10 @@ class FilteringStore(SQLBaseStore): desc="get_user_filter", ) - defer.returnValue(json.loads(def_json)) + defer.returnValue(json.loads(def_json.decode("utf-8"))) def add_user_filter(self, user_localpart, user_filter): - def_json = json.dumps(user_filter) + def_json = json.dumps(user_filter).encode("utf-8") # Need an atomic transaction to SELECT the maximal ID so far then # INSERT a new one -- cgit 1.5.1 From a8945d24d10e74c9011a2ba934799a201d19e12c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 11:07:37 +0100 Subject: Reorder changelog --- CHANGES.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 76060c77cb..da118b7ceb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,18 +1,19 @@ Changes in synapse v0.10.1-rc1 (2015-10-15) =========================================== -* Add CAS support, thanks to Steven Hammerton (PR #295, #296) -* Allow users to view the history of rooms that they have left. (PR #276, #294) -* Bundle in some room state in invites. (PR #275) +* Add support for CAS, thanks to Steven Hammerton (PR #295, #296) +* Add support for using macaroons for ``access_token`` (PR #256, #229) +* Add support for ``m.room.canonical_alias`` (PR #287) +* Add support for viewing the history of rooms that they have left. (PR #276, + #294) +* Add support for refresh tokens (PR #240) * Add flag on creation which disables federation of the room (PR #279) +* Add some room state to invites. (PR #275) * Atomically persist events when joining a room over federation (PR #283) -* Add support for ``m.room.canonical_alias`` (PR #287) * Change default history visibility for private rooms (PR #271) -* Use Macaroons for ``access_token`` (PR #256, #229) * Allow users to redact their own sent events (PR #262) * Use tox for tests (PR #247) * Split up syutil into separate libraries (PR #243) -* Add support for refresh tokens (PR #240) Changes in synapse v0.10.0-r2 (2015-09-16) ========================================== -- cgit 1.5.1 From 22a8c91448f710c20a6aee66ec2a452528f1d637 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 11:19:44 +0100 Subject: Split up run_upgrade --- synapse/storage/schema/delta/24/fts.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/synapse/storage/schema/delta/24/fts.py b/synapse/storage/schema/delta/24/fts.py index b45a5fd820..0c752d8426 100644 --- a/synapse/storage/schema/delta/24/fts.py +++ b/synapse/storage/schema/delta/24/fts.py @@ -55,16 +55,24 @@ CREATE INDEX event_search_ev_ridx ON event_search(room_id); SQLITE_TABLE = ( "CREATE VIRTUAL TABLE event_search USING fts3 ( event_id, room_id, key, value)" ) -SQLITE_INDEX = "CREATE INDEX event_search_ev_idx ON event_search(event_id)" def run_upgrade(cur, database_engine, *args, **kwargs): if isinstance(database_engine, PostgresEngine): - for statement in get_statements(POSTGRES_SQL.splitlines()): - cur.execute(statement) + run_postgres_upgrade(cur) return if isinstance(database_engine, Sqlite3Engine): + run_sqlite_upgrade(cur) + return + + +def run_postgres_upgrade(cur): + for statement in get_statements(POSTGRES_SQL.splitlines()): + cur.execute(statement) + + +def run_sqlite_upgrade(cur): cur.execute(SQLITE_TABLE) rowid = -1 @@ -113,5 +121,3 @@ def run_upgrade(cur, database_engine, *args, **kwargs): " VALUES (?,?,?,?)", rows ) - - # cur.execute(SQLITE_INDEX) -- cgit 1.5.1 From 73260ad01f067495e541a936eef4a14ba2fea5ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 11:24:02 +0100 Subject: Comment on the LIMIT 500 --- synapse/storage/search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 1b987161e2..7d642e18ff 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -54,6 +54,8 @@ class SearchStore(SQLBaseStore): for clause in clauses: sql += " AND " + clause + # We add an arbitrary limit here to ensure we don't try to pull the + # entire table from the database. sql += " ORDER BY rank DESC LIMIT 500" results = yield self._execute( -- cgit 1.5.1 From 3cf9948b8d5956c05026ee734ccf65d203eb6d6b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 11:28:12 +0100 Subject: Add docstring --- synapse/storage/search.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 7d642e18ff..6c10f9631e 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -22,6 +22,17 @@ from synapse.storage.engines import PostgresEngine class SearchStore(SQLBaseStore): @defer.inlineCallbacks def search_msgs(self, room_ids, search_term, keys): + """Performs a full text search over events with give keys. + + Args: + room_ids (list): List of room ids to search in + search_term (str): Search term to search for + keys (list): List of keys to search in, currently supports + "content.body", "content.name", "content.body" + + Returns: + 2-tuple of (dict event_id -> rank, dict event_id -> event) + """ clauses = [] args = [] -- cgit 1.5.1 From b62da463e18a05205725f75508d5053232f1a158 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 11:52:16 +0100 Subject: docstring --- synapse/handlers/search.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 7f1efe2b46..c01c12f8c1 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -82,6 +82,16 @@ class SearchHandler(BaseHandler): @defer.inlineCallbacks def search(self, user, content): + """Performs a full text search for a user. + + Args: + user (UserID) + content (dict): Search parameters + + Returns: + dict to be returned to the client with results of search + """ + try: search_term = content["search_categories"]["room_events"]["search_term"] keys = content["search_categories"]["room_events"].get("keys", [ -- cgit 1.5.1 From edb998ba23cf74de624963f61ca9c897260a3e7e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 14:37:14 +0100 Subject: Explicitly check for Sqlite3Engine --- synapse/storage/search.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 6c10f9631e..dd012fa565 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -16,7 +16,7 @@ from twisted.internet import defer from _base import SQLBaseStore -from synapse.storage.engines import PostgresEngine +from synapse.storage.engines import PostgresEngine, Sqlite3Engine class SearchStore(SQLBaseStore): @@ -56,11 +56,14 @@ class SearchStore(SQLBaseStore): " FROM plainto_tsquery('english', ?) as query, event_search" " WHERE vector @@ query" ) - else: + elif isinstance(self.database_engine, Sqlite3Engine): sql = ( "SELECT 0 as rank, event_id FROM event_search" " WHERE value MATCH ?" ) + else: + # This should be unreachable. + raise Exception("Unrecognized database engine") for clause in clauses: sql += " AND " + clause -- cgit 1.5.1 From f2f031fd57e0ad16c321584bae94487422d89853 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 16 Oct 2015 14:52:08 +0100 Subject: Add config for how many bcrypt rounds to use for password hashes By default we leave it at the default value of 12. But now we can reduce it for preparing users for loadtests or running integration tests. --- synapse/config/registration.py | 6 ++++++ synapse/handlers/auth.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index fa98eced34..f5ef36a9f4 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -33,6 +33,7 @@ class RegistrationConfig(Config): self.registration_shared_secret = config.get("registration_shared_secret") self.macaroon_secret_key = config.get("macaroon_secret_key") + self.bcrypt_rounds = config.get("bcrypt_rounds", 12) def default_config(self, **kwargs): registration_shared_secret = random_string_with_symbols(50) @@ -48,6 +49,11 @@ class RegistrationConfig(Config): registration_shared_secret: "%(registration_shared_secret)s" macaroon_secret_key: "%(macaroon_secret_key)s" + + # Set the number of bcrypt rounds used to generate password hash. + # Larger numbers increase the work factor needed to generate the hash. + # The default number of rounds is 12. + bcrypt_rounds: 12 """ % locals() def add_arguments(self, parser): diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 484f719253..055d395b20 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -44,6 +44,7 @@ class AuthHandler(BaseHandler): LoginType.EMAIL_IDENTITY: self._check_email_identity, LoginType.DUMMY: self._check_dummy_auth, } + self.bcrypt_rounds = hs.config.bcrypt_rounds self.sessions = {} @defer.inlineCallbacks @@ -432,7 +433,7 @@ class AuthHandler(BaseHandler): Returns: Hashed password (str). """ - return bcrypt.hashpw(password, bcrypt.gensalt()) + return bcrypt.hashpw(password, bcrypt.gensalt(self.bcrypt_rounds)) def validate_hash(self, password, stored_hash): """Validates that self.hash(password) == stored_hash. -- cgit 1.5.1 From 366af6b73a02dc85cd59ee17cc2eafb912d221e8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 14:52:48 +0100 Subject: Amalgamate _filter_events_for_client --- synapse/handlers/_base.py | 46 ++++++++++++++++++++++++++++++++++++++ synapse/handlers/message.py | 54 ++++----------------------------------------- synapse/handlers/sync.py | 48 +--------------------------------------- 3 files changed, 51 insertions(+), 97 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index c488ee0f6d..ee2d571329 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -45,6 +45,52 @@ class BaseHandler(object): self.event_builder_factory = hs.get_event_builder_factory() + @defer.inlineCallbacks + def _filter_events_for_client(self, user_id, events): + event_id_to_state = yield self.store.get_state_for_events( + frozenset(e.event_id for e in events), + types=( + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.Member, user_id), + ) + ) + + def allowed(event, state): + if event.type == EventTypes.RoomHistoryVisibility: + return True + + membership_ev = state.get((EventTypes.Member, user_id), None) + if membership_ev: + membership = membership_ev.membership + else: + membership = Membership.LEAVE + + if membership == Membership.JOIN: + return True + + history = state.get((EventTypes.RoomHistoryVisibility, ''), None) + if history: + visibility = history.content.get("history_visibility", "shared") + else: + visibility = "shared" + + if visibility == "public": + return True + elif visibility == "shared": + return True + elif visibility == "joined": + return membership == Membership.JOIN + elif visibility == "invited": + return membership == Membership.INVITE + + return True + + defer.returnValue([ + event + for event in events + if allowed(event, event_id_to_state[event.event_id]) + ]) + def ratelimit(self, user_id): time_now = self.clock.time() allowed, time_allowed = self.ratelimiter.send_message( diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index dfeeae76db..024474d5fe 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -146,7 +146,7 @@ class MessageHandler(BaseHandler): "end": next_token.to_string(), }) - events = yield self._filter_events_for_client(user_id, room_id, events) + events = yield self._filter_events_for_client(user_id, events) time_now = self.clock.time_msec() @@ -161,52 +161,6 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) - @defer.inlineCallbacks - def _filter_events_for_client(self, user_id, room_id, events): - event_id_to_state = yield self.store.get_state_for_events( - frozenset(e.event_id for e in events), - types=( - (EventTypes.RoomHistoryVisibility, ""), - (EventTypes.Member, user_id), - ) - ) - - def allowed(event, state): - if event.type == EventTypes.RoomHistoryVisibility: - return True - - membership_ev = state.get((EventTypes.Member, user_id), None) - if membership_ev: - membership = membership_ev.membership - else: - membership = Membership.LEAVE - - if membership == Membership.JOIN: - return True - - history = state.get((EventTypes.RoomHistoryVisibility, ''), None) - if history: - visibility = history.content.get("history_visibility", "shared") - else: - visibility = "shared" - - if visibility == "public": - return True - elif visibility == "shared": - return True - elif visibility == "joined": - return membership == Membership.JOIN - elif visibility == "invited": - return membership == Membership.INVITE - - return True - - defer.returnValue([ - event - for event in events - if allowed(event, event_id_to_state[event.event_id]) - ]) - @defer.inlineCallbacks def create_and_send_event(self, event_dict, ratelimit=True, token_id=None, txn_id=None): @@ -424,7 +378,7 @@ class MessageHandler(BaseHandler): ).addErrback(unwrapFirstError) messages = yield self._filter_events_for_client( - user_id, event.room_id, messages + user_id, messages ) start_token = now_token.copy_and_replace("room_key", token[0]) @@ -519,7 +473,7 @@ class MessageHandler(BaseHandler): ) messages = yield self._filter_events_for_client( - user_id, room_id, messages + user_id, messages ) start_token = StreamToken(token[0], 0, 0, 0) @@ -599,7 +553,7 @@ class MessageHandler(BaseHandler): ).addErrback(unwrapFirstError) messages = yield self._filter_events_for_client( - user_id, room_id, messages + user_id, messages ) start_token = now_token.copy_and_replace("room_key", token[0]) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 21cf50101a..ee6b881de1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -321,52 +321,6 @@ class SyncHandler(BaseHandler): next_batch=now_token, )) - @defer.inlineCallbacks - def _filter_events_for_client(self, user_id, room_id, events): - event_id_to_state = yield self.store.get_state_for_events( - frozenset(e.event_id for e in events), - types=( - (EventTypes.RoomHistoryVisibility, ""), - (EventTypes.Member, user_id), - ) - ) - - def allowed(event, state): - if event.type == EventTypes.RoomHistoryVisibility: - return True - - membership_ev = state.get((EventTypes.Member, user_id), None) - if membership_ev: - membership = membership_ev.membership - else: - membership = Membership.LEAVE - - if membership == Membership.JOIN: - return True - - history = state.get((EventTypes.RoomHistoryVisibility, ''), None) - if history: - visibility = history.content.get("history_visibility", "shared") - else: - visibility = "shared" - - if visibility == "public": - return True - elif visibility == "shared": - return True - elif visibility == "joined": - return membership == Membership.JOIN - elif visibility == "invited": - return membership == Membership.INVITE - - return True - - defer.returnValue([ - event - for event in events - if allowed(event, event_id_to_state[event.event_id]) - ]) - @defer.inlineCallbacks def load_filtered_recents(self, room_id, sync_config, now_token, since_token=None): @@ -390,7 +344,7 @@ class SyncHandler(BaseHandler): end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter.filter_room_timeline(events) loaded_recents = yield self._filter_events_for_client( - sync_config.user.to_string(), room_id, loaded_recents, + sync_config.user.to_string(), loaded_recents, ) loaded_recents.extend(recents) recents = loaded_recents -- cgit 1.5.1 From b8dd5b1a2d76f0426c600ae19ea9d9612e5327dc Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 16 Oct 2015 14:54:54 +0100 Subject: Verify third party ID server certificates --- synapse/api/auth.py | 11 +++++++++++ synapse/handlers/room.py | 31 +++++++++++++++++++++++++++++-- synapse/http/client.py | 4 ---- synapse/util/third_party_invites.py | 6 +++--- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e96d747b99..aee9b8a14f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -397,13 +397,24 @@ class Auth(object): (EventTypes.ThirdPartyInvite, token,) ) if not invite_event: + logger.info("Failing 3pid invite because no invite found for token %s", token) return False try: public_key = join_third_party_invite["public_key"] key_validity_url = join_third_party_invite["key_validity_url"] if invite_event.content["public_key"] != public_key: + logger.info( + "Failing 3pid invite because public key invite: %s != join: %s", + invite_event.content["public_key"], + public_key + ) return False if invite_event.content["key_validity_url"] != key_validity_url: + logger.info( + "Failing 3pid invite because key_validity_url invite: %s != join: %s", + invite_event.content["key_validity_url"], + key_validity_url + ) return False verify_key = nacl.signing.VerifyKey(decode_base64(public_key)) encoded_signature = join_third_party_invite["signature"] diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 9ffa521aad..3f0cde56f0 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -22,11 +22,16 @@ from synapse.types import UserID, RoomAlias, RoomID from synapse.api.constants import ( EventTypes, Membership, JoinRules, RoomCreationPreset, ) -from synapse.api.errors import StoreError, SynapseError +from synapse.api.errors import AuthError, StoreError, SynapseError from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor +from signedjson.sign import verify_signed_json +from signedjson.key import decode_verify_key_bytes + from collections import OrderedDict +from unpaddedbase64 import decode_base64 + import logging import string @@ -614,12 +619,34 @@ class RoomMemberHandler(BaseHandler): ) if "mxid" in data: - # TODO: Validate the response signature and such + if "signatures" not in data: + raise AuthError(401, "No signatures on 3pid binding") + self.verify_any_signature(data, id_server) defer.returnValue(data["mxid"]) + except IOError as e: logger.warn("Error from identity server lookup: %s" % (e,)) defer.returnValue(None) + @defer.inlineCallbacks + def verify_any_signature(self, data, server_hostname): + if server_hostname not in data["signatures"]: + raise AuthError(401, "No signature from server %s" % (server_hostname,)) + for key_name, signature in data["signatures"][server_hostname].items(): + key_data = yield self.hs.get_simple_http_client().get_json( + "https://%s/_matrix/identity/api/v1/pubkey/%s" % + (server_hostname, key_name,), + ) + if "public_key" not in key_data: + raise AuthError(401, "No public key named %s from %s" % + (key_name, server_hostname,)) + verify_signed_json( + data, + server_hostname, + decode_verify_key_bytes(key_name, decode_base64(key_data["public_key"])) + ) + return + @defer.inlineCallbacks def _make_and_store_3pid_invite( self, diff --git a/synapse/http/client.py b/synapse/http/client.py index 9a5869abee..27e5190224 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -24,7 +24,6 @@ from canonicaljson import encode_canonical_json from twisted.internet import defer, reactor, ssl from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError, - HTTPConnectionPool, ) from twisted.web.http_headers import Headers @@ -59,11 +58,8 @@ class SimpleHttpClient(object): # The default context factory in Twisted 14.0.0 (which we require) is # BrowserLikePolicyForHTTPS which will do regular cert validation # 'like a browser' - pool = HTTPConnectionPool(reactor) - pool.maxPersistentPerHost = 10 self.agent = Agent( reactor, - pool=pool, connectTimeout=15, contextFactory=hs.get_http_client_context_factory() ) diff --git a/synapse/util/third_party_invites.py b/synapse/util/third_party_invites.py index 41e597d5b9..335a9755b2 100644 --- a/synapse/util/third_party_invites.py +++ b/synapse/util/third_party_invites.py @@ -63,7 +63,7 @@ def check_key_valid(http_client, event): event.content["third_party_invite"]["key_validity_url"], {"public_key": event.content["third_party_invite"]["public_key"]} ) - if not response["valid"]: - raise AuthError(403, "Third party certificate was invalid") - except IOError: + except Exception: raise AuthError(502, "Third party certificate could not be checked") + if "valid" not in response or not response["valid"]: + raise AuthError(403, "Third party certificate was invalid") -- cgit 1.5.1 From c225d63e9e50226dce510dda298ad3877460e69a Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 16 Oct 2015 15:07:56 +0100 Subject: Add signing host and keyname to signatures --- synapse/api/auth.py | 14 +++++++++----- synapse/util/third_party_invites.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index aee9b8a14f..5c83aafa7d 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -416,11 +416,15 @@ class Auth(object): key_validity_url ) return False - verify_key = nacl.signing.VerifyKey(decode_base64(public_key)) - encoded_signature = join_third_party_invite["signature"] - signature = decode_base64(encoded_signature) - verify_key.verify(token, signature) - return True + for _, signature_block in join_third_party_invite["signatures"].items(): + for key_name, encoded_signature in signature_block.items(): + if not key_name.startswith("ed25519:"): + return False + verify_key = nacl.signing.VerifyKey(decode_base64(public_key)) + signature = decode_base64(encoded_signature) + verify_key.verify(token, signature) + return True + return False except (KeyError, BadSignatureError,): return False diff --git a/synapse/util/third_party_invites.py b/synapse/util/third_party_invites.py index 335a9755b2..792db5ba39 100644 --- a/synapse/util/third_party_invites.py +++ b/synapse/util/third_party_invites.py @@ -23,7 +23,7 @@ JOIN_KEYS = { "token", "public_key", "key_validity_url", - "signature", + "signatures", "sender", } -- cgit 1.5.1 From d4b5621e0a5edeb66a80d8dd88055a0129def2a9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 15:19:52 +0100 Subject: Remove duplicate _filter_events_for_client --- synapse/handlers/search.py | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index c01c12f8c1..1a5d7381db 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -34,52 +34,6 @@ class SearchHandler(BaseHandler): def __init__(self, hs): super(SearchHandler, self).__init__(hs) - @defer.inlineCallbacks - def _filter_events_for_client(self, user_id, events): - event_id_to_state = yield self.store.get_state_for_events( - frozenset(e.event_id for e in events), - types=( - (EventTypes.RoomHistoryVisibility, ""), - (EventTypes.Member, user_id), - ) - ) - - def allowed(event, state): - if event.type == EventTypes.RoomHistoryVisibility: - return True - - membership_ev = state.get((EventTypes.Member, user_id), None) - if membership_ev: - membership = membership_ev.membership - else: - membership = Membership.LEAVE - - if membership == Membership.JOIN: - return True - - history = state.get((EventTypes.RoomHistoryVisibility, ''), None) - if history: - visibility = history.content.get("history_visibility", "shared") - else: - visibility = "shared" - - if visibility == "public": - return True - elif visibility == "shared": - return True - elif visibility == "joined": - return membership == Membership.JOIN - elif visibility == "invited": - return membership == Membership.INVITE - - return True - - defer.returnValue([ - event - for event in events - if allowed(event, event_id_to_state[event.event_id]) - ]) - @defer.inlineCallbacks def search(self, user, content): """Performs a full text search for a user. -- cgit 1.5.1 From fc012aa8dc7a3c05c1824a2c333d6c73ebc8726e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 16 Oct 2015 15:28:43 +0100 Subject: Fix FilteringStore.get_user_filter to work with postgres --- synapse/storage/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index 700da8d8c3..fcd43c7fdd 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -34,7 +34,7 @@ class FilteringStore(SQLBaseStore): desc="get_user_filter", ) - defer.returnValue(json.loads(def_json.decode("utf-8"))) + defer.returnValue(json.loads(str(def_json).decode("utf-8"))) def add_user_filter(self, user_localpart, user_filter): def_json = json.dumps(user_filter).encode("utf-8") -- cgit 1.5.1 From 380f148db7d710ece7679e207334483bda407aa5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 15:32:51 +0100 Subject: Remove unused import --- synapse/handlers/search.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 1a5d7381db..22808b9c07 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -17,9 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.constants import ( - EventTypes, Membership, -) +from synapse.api.constants import Membership from synapse.api.errors import SynapseError from synapse.events.utils import serialize_event -- cgit 1.5.1 From f2d698cb52883d8d43faabefdc70e2ade9ebb8b8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 16:46:48 +0100 Subject: Typing --- synapse/storage/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index dd012fa565..a3c69c5ab3 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -22,13 +22,13 @@ from synapse.storage.engines import PostgresEngine, Sqlite3Engine class SearchStore(SQLBaseStore): @defer.inlineCallbacks def search_msgs(self, room_ids, search_term, keys): - """Performs a full text search over events with give keys. + """Performs a full text search over events with given keys. Args: room_ids (list): List of room ids to search in search_term (str): Search term to search for keys (list): List of keys to search in, currently supports - "content.body", "content.name", "content.body" + "content.body", "content.name", "content.topic" Returns: 2-tuple of (dict event_id -> rank, dict event_id -> event) -- cgit 1.5.1 From 46d39343d976a933c3f2dfd19e5e552c01c93bf4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Oct 2015 16:58:00 +0100 Subject: Explicitly check for Sqlite3Engine --- synapse/storage/room.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 0527cee05d..13441fcdce 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -19,7 +19,7 @@ from synapse.api.errors import StoreError from ._base import SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks -from .engines import PostgresEngine +from .engines import PostgresEngine, Sqlite3Engine import collections import logging @@ -208,11 +208,14 @@ class RoomStore(SQLBaseStore): "INSERT INTO event_search (event_id, room_id, key, vector)" " VALUES (?,?,?,to_tsvector('english', ?))" ) - else: + elif isinstance(self.database_engine, Sqlite3Engine): sql = ( "INSERT INTO event_search (event_id, room_id, key, value)" " VALUES (?,?,?,?)" ) + else: + # This should be unreachable. + raise Exception("Unrecognized database engine") txn.execute(sql, (event.event_id, event.room_id, key, value,)) -- cgit 1.5.1 From 0e5239ffc38c6c13799c0001f2267fe8290a7300 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Fri, 16 Oct 2015 17:45:48 +0100 Subject: Stuff signed data in a standalone object Makes both generating it in sydent, and verifying it here, simpler at the cost of some repetition --- synapse/api/auth.py | 21 ++++++++++++++------- synapse/util/third_party_invites.py | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 5c83aafa7d..cf19eda4e9 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -14,7 +14,8 @@ # limitations under the License. """This module contains classes for authenticating the user.""" -from nacl.exceptions import BadSignatureError +from signedjson.key import decode_verify_key_bytes +from signedjson.sign import verify_signed_json, SignatureVerifyException from twisted.internet import defer @@ -26,7 +27,6 @@ from synapse.util import third_party_invites from unpaddedbase64 import decode_base64 import logging -import nacl.signing import pymacaroons logger = logging.getLogger(__name__) @@ -416,16 +416,23 @@ class Auth(object): key_validity_url ) return False - for _, signature_block in join_third_party_invite["signatures"].items(): + signed = join_third_party_invite["signed"] + if signed["mxid"] != event.user_id: + return False + if signed["token"] != token: + return False + for server, signature_block in signed["signatures"].items(): for key_name, encoded_signature in signature_block.items(): if not key_name.startswith("ed25519:"): return False - verify_key = nacl.signing.VerifyKey(decode_base64(public_key)) - signature = decode_base64(encoded_signature) - verify_key.verify(token, signature) + verify_key = decode_verify_key_bytes( + key_name, + decode_base64(public_key) + ) + verify_signed_json(signed, server, verify_key) return True return False - except (KeyError, BadSignatureError,): + except (KeyError, SignatureVerifyException,): return False def _get_power_level_event(self, auth_events): diff --git a/synapse/util/third_party_invites.py b/synapse/util/third_party_invites.py index 792db5ba39..31d186740d 100644 --- a/synapse/util/third_party_invites.py +++ b/synapse/util/third_party_invites.py @@ -23,8 +23,8 @@ JOIN_KEYS = { "token", "public_key", "key_validity_url", - "signatures", "sender", + "signed", } -- cgit 1.5.1 From 243a79d291382bf8d7dd476c41e03ca4cb702dd2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 16 Oct 2015 18:25:19 +0100 Subject: Surely we don't need to preserve 'events_default' twice --- synapse/events/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index b36eec0993..48548f8c40 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -66,7 +66,6 @@ def prune_event(event): "users_default", "events", "events_default", - "events_default", "state_default", "ban", "kick", -- cgit 1.5.1 From aff4d850bdc5d6108b1f6f84591b44db6e496d75 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 16 Oct 2015 19:56:46 +0100 Subject: Add some unit tests of prune_events() --- tests/events/__init__.py | 0 tests/events/test_utils.py | 115 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 tests/events/__init__.py create mode 100644 tests/events/test_utils.py diff --git a/tests/events/__init__.py b/tests/events/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py new file mode 100644 index 0000000000..16179921f0 --- /dev/null +++ b/tests/events/test_utils.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import unittest + +from synapse.events import FrozenEvent +from synapse.events.utils import prune_event + +class PruneEventTestCase(unittest.TestCase): + """ Asserts that a new event constructed with `evdict` will look like + `matchdict` when it is redacted. """ + def run_test(self, evdict, matchdict): + self.assertEquals( + prune_event(FrozenEvent(evdict)).get_dict(), + matchdict + ) + + def test_minimal(self): + self.run_test( + {'type': 'A'}, + { + 'type': 'A', + 'content': {}, + 'signatures': {}, + 'unsigned': {}, + } + ) + + def test_basic_keys(self): + self.run_test( + { + 'type': 'A', + 'room_id': '!1:domain', + 'sender': '@2:domain', + 'event_id': '$3:domain', + 'origin': 'domain', + }, + { + 'type': 'A', + 'room_id': '!1:domain', + 'sender': '@2:domain', + 'event_id': '$3:domain', + 'origin': 'domain', + 'content': {}, + 'signatures': {}, + 'unsigned': {}, + } + ) + + def test_unsigned_age_ts(self): + self.run_test( + { + 'type': 'B', + 'unsigned': {'age_ts': 20}, + }, + { + 'type': 'B', + 'content': {}, + 'signatures': {}, + 'unsigned': {'age_ts': 20}, + } + ) + + self.run_test( + { + 'type': 'B', + 'unsigned': {'other_key': 'here'}, + }, + { + 'type': 'B', + 'content': {}, + 'signatures': {}, + 'unsigned': {}, + } + ) + + def test_content(self): + self.run_test( + { + 'type': 'C', + 'content': {'things': 'here'}, + }, + { + 'type': 'C', + 'content': {}, + 'signatures': {}, + 'unsigned': {}, + } + ) + + self.run_test( + { + 'type': 'm.room.create', + 'content': {'creator': '@2:domain', 'other_field': 'here'}, + }, + { + 'type': 'm.room.create', + 'content': {'creator': '@2:domain'}, + 'signatures': {}, + 'unsigned': {}, + } + ) -- cgit 1.5.1 From 0aab34004b2e56c3ab79f514be264c568ad71fd3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 19 Oct 2015 14:40:15 +0100 Subject: Initial minimial hack at a test of event hashing and signing --- tests/crypto/test_event_signing.py | 98 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 tests/crypto/test_event_signing.py diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py new file mode 100644 index 0000000000..0b560e9317 --- /dev/null +++ b/tests/crypto/test_event_signing.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest +from tests.utils import MockClock + +from synapse.events.builder import EventBuilderFactory +from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.types import EventID + +from unpaddedbase64 import decode_base64 + +import nacl.signing + + +# Perform these tests using given secret key so we get entirely deterministic +# signatures output that we can test against. +SIGNING_KEY_SEED = decode_base64( + "YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1" +) + +KEY_ALG = "ed25519" +KEY_VER = 1 +KEY_NAME = "%s:%d" % (KEY_ALG, KEY_VER) + +HOSTNAME = "domain" + + +class EventBuilderFactoryWithPredicableIDs(EventBuilderFactory): + """ A subclass of EventBuilderFactory that generates entirely predicatable + event IDs, so we can assert on them. """ + def create_event_id(self): + i = str(self.event_id_count) + self.event_id_count += 1 + + return EventID.create(i, self.hostname).to_string() + + +class EventSigningTestCase(unittest.TestCase): + + def setUp(self): + self.event_builder_factory = EventBuilderFactoryWithPredicableIDs( + clock=MockClock(), + hostname=HOSTNAME, + ) + + self.signing_key = nacl.signing.SigningKey(SIGNING_KEY_SEED) + self.signing_key.alg = KEY_ALG + self.signing_key.version = KEY_VER + + def test_sign(self): + builder = self.event_builder_factory.new( + {'type': "X"} + ) + self.assertEquals( + builder.build().get_dict(), + { + 'event_id': "$0:domain", + 'origin': "domain", + 'origin_server_ts': 1000000, + 'signatures': {}, + 'type': "X", + 'unsigned': {'age_ts': 1000000}, + }, + ) + + add_hashes_and_signatures(builder, HOSTNAME, self.signing_key) + + event = builder.build() + + self.assertTrue(hasattr(event, 'hashes')) + self.assertTrue('sha256' in event.hashes) + self.assertEquals( + event.hashes['sha256'], + "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI", + ) + + self.assertTrue(hasattr(event, 'signatures')) + self.assertTrue(HOSTNAME in event.signatures) + self.assertTrue(KEY_NAME in event.signatures["domain"]) + self.assertEquals( + event.signatures[HOSTNAME][KEY_NAME], + "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+" + "aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA", + ) -- cgit 1.5.1 From 07b58a431f9e0367f8c08d2bc8983473c8a0c379 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 19 Oct 2015 15:00:52 +0100 Subject: Another signing test vector using an 'm.room.message' with content, so that the implementation will have to redact it --- tests/crypto/test_event_signing.py | 50 +++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 0b560e9317..0f487d9c7b 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -61,7 +61,7 @@ class EventSigningTestCase(unittest.TestCase): self.signing_key.alg = KEY_ALG self.signing_key.version = KEY_VER - def test_sign(self): + def test_sign_minimal(self): builder = self.event_builder_factory.new( {'type': "X"} ) @@ -96,3 +96,51 @@ class EventSigningTestCase(unittest.TestCase): "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+" "aIbygsSdLOFzvdDjww8zUVKCmI02eP9xtyJxc/cLiBA", ) + + def test_sign_message(self): + builder = self.event_builder_factory.new( + { + 'type': "m.room.message", + 'sender': "@u:domain", + 'room_id': "!r:domain", + 'content': { + 'body': "Here is the message content", + }, + } + ) + self.assertEquals( + builder.build().get_dict(), + { + 'content': { + 'body': "Here is the message content", + }, + 'event_id': "$0:domain", + 'origin': "domain", + 'origin_server_ts': 1000000, + 'type': "m.room.message", + 'room_id': "!r:domain", + 'sender': "@u:domain", + 'signatures': {}, + 'unsigned': {'age_ts': 1000000}, + } + ) + + add_hashes_and_signatures(builder, HOSTNAME, self.signing_key) + + event = builder.build() + + self.assertTrue(hasattr(event, 'hashes')) + self.assertTrue('sha256' in event.hashes) + self.assertEquals( + event.hashes['sha256'], + "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g", + ) + + self.assertTrue(hasattr(event, 'signatures')) + self.assertTrue(HOSTNAME in event.signatures) + self.assertTrue(KEY_NAME in event.signatures["domain"]) + self.assertEquals( + event.signatures[HOSTNAME][KEY_NAME], + "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUw" + "u6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA" + ) -- cgit 1.5.1 From a8795c9644d555e95a6be3211b4e79e447087697 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 19 Oct 2015 15:24:49 +0100 Subject: Use assertIn() instead of assertTrue on the 'in' operator --- tests/crypto/test_event_signing.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 0f487d9c7b..010fe4ed33 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -82,15 +82,15 @@ class EventSigningTestCase(unittest.TestCase): event = builder.build() self.assertTrue(hasattr(event, 'hashes')) - self.assertTrue('sha256' in event.hashes) + self.assertIn('sha256', event.hashes) self.assertEquals( event.hashes['sha256'], "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI", ) self.assertTrue(hasattr(event, 'signatures')) - self.assertTrue(HOSTNAME in event.signatures) - self.assertTrue(KEY_NAME in event.signatures["domain"]) + self.assertIn(HOSTNAME, event.signatures) + self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEquals( event.signatures[HOSTNAME][KEY_NAME], "2Wptgo4CwmLo/Y8B8qinxApKaCkBG2fjTWB7AbP5Uy+" @@ -130,15 +130,15 @@ class EventSigningTestCase(unittest.TestCase): event = builder.build() self.assertTrue(hasattr(event, 'hashes')) - self.assertTrue('sha256' in event.hashes) + self.assertIn('sha256', event.hashes) self.assertEquals( event.hashes['sha256'], "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g", ) self.assertTrue(hasattr(event, 'signatures')) - self.assertTrue(HOSTNAME in event.signatures) - self.assertTrue(KEY_NAME in event.signatures["domain"]) + self.assertIn(HOSTNAME, event.signatures) + self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEquals( event.signatures[HOSTNAME][KEY_NAME], "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUw" -- cgit 1.5.1 From 68b7fc3e2ba0aae7813b0bae52370860b5cd9f26 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 19 Oct 2015 17:26:18 +0100 Subject: Add rooms that the user has left under archived in v2 sync. --- synapse/handlers/sync.py | 128 ++++++++++++++++++++++++++++++++++- synapse/rest/client/v2_alpha/sync.py | 29 ++++++-- synapse/storage/roommember.py | 13 ++++ 3 files changed, 161 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ee6b881de1..1891cd088c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -61,18 +61,37 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [ return bool(self.timeline or self.state or self.ephemeral) +class ArchivedSyncResult(collections.namedtuple("JoinedSyncResult", [ + "room_id", + "timeline", + "state", +])): + __slots__ = [] + + def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if room needs to be part of the sync result. + """ + return bool(self.timeline or self.state) + + class InvitedSyncResult(collections.namedtuple("InvitedSyncResult", [ "room_id", "invite", ])): __slots__ = [] + def __nonzero__(self): + """Invited rooms should always be reported to the client""" + return True + class SyncResult(collections.namedtuple("SyncResult", [ "next_batch", # Token for the next sync "presence", # List of presence events for the user. "joined", # JoinedSyncResult for each joined room. "invited", # InvitedSyncResult for each invited room. + "archived", # ArchivedSyncResult for each archived room. ])): __slots__ = [] @@ -156,11 +175,14 @@ class SyncHandler(BaseHandler): ) room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=sync_config.user.to_string(), - membership_list=[Membership.INVITE, Membership.JOIN] + membership_list=[ + Membership.INVITE, Membership.JOIN, Membership.LEAVE + ] ) joined = [] invited = [] + archived = [] for event in room_list: if event.membership == Membership.JOIN: room_sync = yield self.initial_sync_for_joined_room( @@ -173,11 +195,23 @@ class SyncHandler(BaseHandler): room_id=event.room_id, invite=invite, )) + elif event.membership == Membership.LEAVE: + leave_token = now_token.copy_and_replace( + "room_key", "s%d" % (event.stream_ordering,) + ) + room_sync = yield self.initial_sync_for_archived_room( + sync_config=sync_config, + room_id=event.room_id, + leave_event_id=event.event_id, + leave_token=leave_token, + ) + archived.append(room_sync) defer.returnValue(SyncResult( presence=presence, joined=joined, invited=invited, + archived=archived, next_batch=now_token, )) @@ -204,6 +238,28 @@ class SyncHandler(BaseHandler): ephemeral=[], )) + @defer.inlineCallbacks + def initial_sync_for_archived_room(self, room_id, sync_config, + leave_event_id, leave_token): + """Sync a room for a client which is starting without any state + Returns: + A Deferred JoinedSyncResult. + """ + + batch = yield self.load_filtered_recents( + room_id, sync_config, leave_token, + ) + + leave_state = yield self.store.get_state_for_events( + [leave_event_id], None + ) + + defer.returnValue(ArchivedSyncResult( + room_id=room_id, + timeline=batch, + state=leave_state[leave_event_id].values(), + )) + @defer.inlineCallbacks def incremental_sync_with_gap(self, sync_config, since_token): """ Get the incremental delta needed to bring the client up to @@ -257,18 +313,22 @@ class SyncHandler(BaseHandler): ) joined = [] + archived = [] if len(room_events) <= timeline_limit: # There is no gap in any of the rooms. Therefore we can just # partition the new events by room and return them. invite_events = [] + leave_events = [] events_by_room_id = {} for event in room_events: events_by_room_id.setdefault(event.room_id, []).append(event) if event.room_id not in joined_room_ids: if (event.type == EventTypes.Member - and event.membership == Membership.INVITE and event.state_key == sync_config.user.to_string()): - invite_events.append(event) + if event.membership == Membership.INVITE: + invite_events.append(event) + elif event.membership == Membership.LEAVE: + leave_events.append(event) for room_id in joined_room_ids: recents = events_by_room_id.get(room_id, []) @@ -296,11 +356,16 @@ class SyncHandler(BaseHandler): ) if room_sync: joined.append(room_sync) + else: invite_events = yield self.store.get_invites_for_user( sync_config.user.to_string() ) + leave_events = yield self.store.get_leave_events_for_user( + sync_config.user.to_string() + ) + for room_id in joined_room_ids: room_sync = yield self.incremental_sync_with_gap_for_room( room_id, sync_config, since_token, now_token, @@ -309,6 +374,12 @@ class SyncHandler(BaseHandler): if room_sync: joined.append(room_sync) + for leave_event in leave_events: + room_sync = yield self.incremental_sync_for_archived_room( + sync_config, leave_event, since_token + ) + archived.append(room_sync) + invited = [ InvitedSyncResult(room_id=event.room_id, invite=event) for event in invite_events @@ -318,6 +389,7 @@ class SyncHandler(BaseHandler): presence=presence, joined=joined, invited=invited, + archived=archived, next_batch=now_token, )) @@ -416,6 +488,56 @@ class SyncHandler(BaseHandler): defer.returnValue(room_sync) + @defer.inlineCallbacks + def incremental_sync_for_archived_room(self, sync_config, leave_event, + since_token): + """ Get the incremental delta needed to bring the client up to date for + the archived room. + Returns: + A Deferred ArchivedSyncResult + """ + + stream_token = yield self.store.get_stream_token_for_event( + leave_event.event_id + ) + + leave_token = since_token.copy_and_replace("room_key", stream_token) + + batch = yield self.load_filtered_recents( + leave_event.room_id, sync_config, leave_token, since_token, + ) + + logging.debug("Recents %r", batch) + + # TODO(mjark): This seems racy since this isn't being passed a + # token to indicate what point in the stream this is + leave_state = yield self.store.get_state_for_events( + [leave_event.event_id], None + ) + + state_events_at_leave = leave_state[leave_event.event_id].values() + + state_at_previous_sync = yield self.get_state_at_previous_sync( + leave_event.room_id, since_token=since_token + ) + + state_events_delta = yield self.compute_state_delta( + since_token=since_token, + previous_state=state_at_previous_sync, + current_state=state_events_at_leave, + ) + + room_sync = ArchivedSyncResult( + room_id=leave_event.room_id, + timeline=batch, + state=state_events_delta, + ) + + logging.debug("Room sync: %r", room_sync) + + defer.returnValue(room_sync) + + @defer.inlineCallbacks def get_state_at_previous_sync(self, room_id, since_token): """ Get the room state at the previous sync the client made. diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index fffecb24f5..73473a7e6b 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -136,6 +136,10 @@ class SyncRestServlet(RestServlet): sync_result.invited, filter, time_now, token_id ) + archived = self.encode_archived( + sync_result.archived, filter, time_now, token_id + ) + response_content = { "presence": self.encode_presence( sync_result.presence, filter, time_now @@ -143,7 +147,7 @@ class SyncRestServlet(RestServlet): "rooms": { "joined": joined, "invited": invited, - "archived": {}, + "archived": archived, }, "next_batch": sync_result.next_batch.to_string(), } @@ -182,14 +186,20 @@ class SyncRestServlet(RestServlet): return invited + def encode_archived(self, rooms, filter, time_now, token_id): + joined = {} + for room in rooms: + joined[room.room_id] = self.encode_room( + room, filter, time_now, token_id, joined=False + ) + + return joined + @staticmethod - def encode_room(room, filter, time_now, token_id): + def encode_room(room, filter, time_now, token_id, joined=True): event_map = {} state_events = filter.filter_room_state(room.state) - timeline_events = filter.filter_room_timeline(room.timeline.events) - ephemeral_events = filter.filter_room_ephemeral(room.ephemeral) state_event_ids = [] - timeline_event_ids = [] for event in state_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( @@ -198,6 +208,8 @@ class SyncRestServlet(RestServlet): ) state_event_ids.append(event.event_id) + timeline_events = filter.filter_room_timeline(room.timeline.events) + timeline_event_ids = [] for event in timeline_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( @@ -205,6 +217,7 @@ class SyncRestServlet(RestServlet): event_format=format_event_for_client_v2_without_event_id, ) timeline_event_ids.append(event.event_id) + result = { "event_map": event_map, "timeline": { @@ -213,8 +226,12 @@ class SyncRestServlet(RestServlet): "limited": room.timeline.limited, }, "state": {"events": state_event_ids}, - "ephemeral": {"events": ephemeral_events}, } + + if joined: + ephemeral_events = filter.filter_room_ephemeral(room.ephemeral) + result["ephemeral"] = {"events": ephemeral_events} + return result diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index dd98dcfda8..623400fd36 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -124,6 +124,19 @@ class RoomMemberStore(SQLBaseStore): invites.event_id for invite in invites ])) + def get_leave_events_for_user(self, user_id): + """ Get all the leave events for a user + Args: + user_id (str): The user ID. + Returns: + A deferred list of event objects. + """ + return self.get_rooms_for_user_where_membership_is( + user_id, [Membership.LEAVE] + ).addCallback(lambda leaves: self._get_events([ + leave.event_id for leave in leaves + ])) + def get_rooms_for_user_where_membership_is(self, user_id, membership_list): """ Get all the rooms for this user where the membership for this user matches one in the membership list. -- cgit 1.5.1 From 531e3aa75effdec137c1ffbdb1fb0e8cb0cbe40e Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 19 Oct 2015 17:37:35 +0100 Subject: Capture __init__.py --- tests/crypto/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/crypto/__init__.py diff --git a/tests/crypto/__init__.py b/tests/crypto/__init__.py new file mode 100644 index 0000000000..9bff9ec169 --- /dev/null +++ b/tests/crypto/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- cgit 1.5.1 From 9ed784098a94cf80d2582cc1d98484ac9d748eee Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 19 Oct 2015 17:42:34 +0100 Subject: Invoke EventBuilder directly instead of going via the EventBuilderFactory --- tests/crypto/test_event_signing.py | 38 +++----------------------------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 010fe4ed33..7913472941 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -15,11 +15,9 @@ from tests import unittest -from tests.utils import MockClock -from synapse.events.builder import EventBuilderFactory +from synapse.events.builder import EventBuilder from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.types import EventID from unpaddedbase64 import decode_base64 @@ -39,34 +37,15 @@ KEY_NAME = "%s:%d" % (KEY_ALG, KEY_VER) HOSTNAME = "domain" -class EventBuilderFactoryWithPredicableIDs(EventBuilderFactory): - """ A subclass of EventBuilderFactory that generates entirely predicatable - event IDs, so we can assert on them. """ - def create_event_id(self): - i = str(self.event_id_count) - self.event_id_count += 1 - - return EventID.create(i, self.hostname).to_string() - - class EventSigningTestCase(unittest.TestCase): def setUp(self): - self.event_builder_factory = EventBuilderFactoryWithPredicableIDs( - clock=MockClock(), - hostname=HOSTNAME, - ) - self.signing_key = nacl.signing.SigningKey(SIGNING_KEY_SEED) self.signing_key.alg = KEY_ALG self.signing_key.version = KEY_VER def test_sign_minimal(self): - builder = self.event_builder_factory.new( - {'type': "X"} - ) - self.assertEquals( - builder.build().get_dict(), + builder = EventBuilder( { 'event_id': "$0:domain", 'origin': "domain", @@ -98,18 +77,7 @@ class EventSigningTestCase(unittest.TestCase): ) def test_sign_message(self): - builder = self.event_builder_factory.new( - { - 'type': "m.room.message", - 'sender': "@u:domain", - 'room_id': "!r:domain", - 'content': { - 'body': "Here is the message content", - }, - } - ) - self.assertEquals( - builder.build().get_dict(), + builder = EventBuilder( { 'content': { 'body': "Here is the message content", -- cgit 1.5.1 From 51d03e65b2d481a59dfb08e6c75aa349fca71fe6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 19 Oct 2015 17:48:58 +0100 Subject: Fix pep8 --- synapse/handlers/sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1891cd088c..5ca2606443 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -537,7 +537,6 @@ class SyncHandler(BaseHandler): defer.returnValue(room_sync) - @defer.inlineCallbacks def get_state_at_previous_sync(self, room_id, since_token): """ Get the room state at the previous sync the client made. -- cgit 1.5.1 From b02a342750f84ffebb793aa5d3c80780684dd147 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 20 Oct 2015 11:07:50 +0100 Subject: Don't 500 when the email doesn't map to a valid user ID. --- synapse/rest/client/v1/login.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index dacc416055..b2e4cb8eaa 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -101,6 +101,10 @@ class LoginRestServlet(ClientV1RestServlet): user_id = yield self.hs.get_datastore().get_user_id_by_threepid( login_submission['medium'], login_submission['address'] ) + if not user_id: + raise LoginError( + 401, "Unrecognised address", errcode=Codes.UNAUTHORIZED + ) else: user_id = login_submission['user'] -- cgit 1.5.1 From 137fafce4ee06e76b05d37807611e10055059f62 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 20 Oct 2015 11:58:58 +0100 Subject: Allow rejecting invites This is done by using the same /leave flow as you would use if you had already accepted the invite and wanted to leave. --- synapse/api/auth.py | 6 +- synapse/federation/federation_client.py | 67 +++++++++- synapse/federation/federation_server.py | 14 +++ synapse/federation/transport/client.py | 24 +++- synapse/federation/transport/server.py | 20 +++ synapse/handlers/federation.py | 209 +++++++++++++++++++++++++------- synapse/handlers/room.py | 102 +++++++++------- tests/rest/client/v1/test_rooms.py | 4 +- 8 files changed, 353 insertions(+), 93 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index cf19eda4e9..494c8ac3d4 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -308,7 +308,11 @@ class Auth(object): ) if Membership.JOIN != membership: - # JOIN is the only action you can perform if you're not in the room + if (caller_invited + and Membership.LEAVE == membership + and target_user_id == event.user_id): + return True + if not caller_in_room: # caller isn't joined raise AuthError( 403, diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index f5b430e046..723f571284 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -17,6 +17,7 @@ from twisted.internet import defer from .federation_base import FederationBase +from synapse.api.constants import Membership from .units import Edu from synapse.api.errors import ( @@ -357,7 +358,34 @@ class FederationClient(FederationBase): defer.returnValue(signed_auth) @defer.inlineCallbacks - def make_join(self, destinations, room_id, user_id, content): + def make_membership_event(self, destinations, room_id, user_id, membership, content): + """ + Creates an m.room.member event, with context, without participating in the room. + + Does so by asking one of the already participating servers to create an + event with proper context. + + Note that this does not append any events to any graphs. + + Args: + destinations (str): Candidate homeservers which are probably + participating in the room. + room_id (str): The room in which the event will happen. + user_id (str): The user whose membership is being evented. + membership (str): The "membership" property of the event. Must be + one of "join" or "leave". + content (object): Any additional data to put into the content field + of the event. + Return: + A tuple of (origin (str), event (object)) where origin is the remote + homeserver which generated the event. + """ + valid_memberships = {Membership.JOIN, Membership.LEAVE} + if membership not in valid_memberships: + raise RuntimeError( + "make_membership_event called with membership='%s', must be one of %s" % + (membership, ",".join(valid_memberships)) + ) for destination in destinations: if destination == self.server_name: continue @@ -368,13 +396,13 @@ class FederationClient(FederationBase): content["third_party_invite"] ) try: - ret = yield self.transport_layer.make_join( - destination, room_id, user_id, args + ret = yield self.transport_layer.make_membership_event( + destination, room_id, user_id, membership, args ) pdu_dict = ret["event"] - logger.debug("Got response to make_join: %s", pdu_dict) + logger.debug("Got response to make_%s: %s", membership, pdu_dict) defer.returnValue( (destination, self.event_from_pdu_json(pdu_dict)) @@ -384,8 +412,8 @@ class FederationClient(FederationBase): raise except Exception as e: logger.warn( - "Failed to make_join via %s: %s", - destination, e.message + "Failed to make_%s via %s: %s", + membership, destination, e.message ) raise RuntimeError("Failed to send to any server.") @@ -491,6 +519,33 @@ class FederationClient(FederationBase): defer.returnValue(pdu) + @defer.inlineCallbacks + def send_leave(self, destinations, pdu): + for destination in destinations: + if destination == self.server_name: + continue + + try: + time_now = self._clock.time_msec() + _, content = yield self.transport_layer.send_leave( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=pdu.get_pdu_json(time_now), + ) + + logger.debug("Got content: %s", content) + defer.returnValue(None) + except CodeMessageException: + raise + except Exception as e: + logger.exception( + "Failed to send_leave via %s: %s", + destination, e.message + ) + + raise RuntimeError("Failed to send to any server.") + @defer.inlineCallbacks def query_auth(self, destination, room_id, event_id, local_auth): """ diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 7934f740e0..9e2d9ee74c 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -267,6 +267,20 @@ class FederationServer(FederationBase): ], })) + @defer.inlineCallbacks + def on_make_leave_request(self, room_id, user_id): + pdu = yield self.handler.on_make_leave_request(room_id, user_id) + time_now = self._clock.time_msec() + defer.returnValue({"event": pdu.get_pdu_json(time_now)}) + + @defer.inlineCallbacks + def on_send_leave_request(self, origin, content): + logger.debug("on_send_leave_request: content: %s", content) + pdu = self.event_from_pdu_json(content) + logger.debug("on_send_leave_request: pdu sigs: %s", pdu.signatures) + yield self.handler.on_send_leave_request(origin, pdu) + defer.returnValue((200, {})) + @defer.inlineCallbacks def on_event_auth(self, origin, room_id, event_id): time_now = self._clock.time_msec() diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index ae4195e83a..a81b3c4345 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -14,6 +14,7 @@ # limitations under the License. from twisted.internet import defer +from synapse.api.constants import Membership from synapse.api.urls import FEDERATION_PREFIX as PREFIX from synapse.util.logutils import log_function @@ -160,8 +161,14 @@ class TransportLayerClient(object): @defer.inlineCallbacks @log_function - def make_join(self, destination, room_id, user_id, args={}): - path = PREFIX + "/make_join/%s/%s" % (room_id, user_id) + def make_membership_event(self, destination, room_id, user_id, membership, args={}): + valid_memberships = {Membership.JOIN, Membership.LEAVE} + if membership not in valid_memberships: + raise RuntimeError( + "make_membership_event called with membership='%s', must be one of %s" % + (membership, ",".join(valid_memberships)) + ) + path = PREFIX + "/make_%s/%s/%s" % (membership, room_id, user_id) content = yield self.client.get_json( destination=destination, @@ -185,6 +192,19 @@ class TransportLayerClient(object): defer.returnValue(response) + @defer.inlineCallbacks + @log_function + def send_leave(self, destination, room_id, event_id, content): + path = PREFIX + "/send_leave/%s/%s" % (room_id, event_id) + + response = yield self.client.put_json( + destination=destination, + path=path, + data=content, + ) + + defer.returnValue(response) + @defer.inlineCallbacks @log_function def send_invite(self, destination, room_id, event_id, content): diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 6e394f039e..8184159210 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -296,6 +296,24 @@ class FederationMakeJoinServlet(BaseFederationServlet): defer.returnValue((200, content)) +class FederationMakeLeaveServlet(BaseFederationServlet): + PATH = "/make_leave/([^/]*)/([^/]*)" + + @defer.inlineCallbacks + def on_GET(self, origin, content, query, context, user_id): + content = yield self.handler.on_make_leave_request(context, user_id) + defer.returnValue((200, content)) + + +class FederationSendLeaveServlet(BaseFederationServlet): + PATH = "/send_leave/([^/]*)/([^/]*)" + + @defer.inlineCallbacks + def on_PUT(self, origin, content, query, room_id, txid): + content = yield self.handler.on_send_leave_request(origin, content) + defer.returnValue((200, content)) + + class FederationEventAuthServlet(BaseFederationServlet): PATH = "/event_auth/([^/]*)/([^/]*)" @@ -385,8 +403,10 @@ SERVLET_CLASSES = ( FederationBackfillServlet, FederationQueryServlet, FederationMakeJoinServlet, + FederationMakeLeaveServlet, FederationEventServlet, FederationSendJoinServlet, + FederationSendLeaveServlet, FederationInviteServlet, FederationQueryAuthServlet, FederationGetMissingEventsServlet, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 946ff97c7d..ae9d227586 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -565,7 +565,7 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks - def do_invite_join(self, target_hosts, room_id, joinee, content, snapshot): + def do_invite_join(self, target_hosts, room_id, joinee, content): """ Attempts to join the `joinee` to the room `room_id` via the server `target_host`. @@ -581,50 +581,19 @@ class FederationHandler(BaseHandler): yield self.store.clean_room_for_join(room_id) - origin, pdu = yield self.replication_layer.make_join( + origin, event = yield self._make_and_verify_event( target_hosts, room_id, joinee, + "join", content ) - logger.debug("Got response to make_join: %s", pdu) - - event = pdu - - # We should assert some things. - # FIXME: Do this in a nicer way - assert(event.type == EventTypes.Member) - assert(event.user_id == joinee) - assert(event.state_key == joinee) - assert(event.room_id == room_id) - - event.internal_metadata.outlier = False - self.room_queues[room_id] = [] - - builder = self.event_builder_factory.new( - unfreeze(event.get_pdu_json()) - ) - handled_events = set() try: - builder.event_id = self.event_builder_factory.create_event_id() - builder.origin = self.hs.hostname - builder.content = content - - if not hasattr(event, "signatures"): - builder.signatures = {} - - add_hashes_and_signatures( - builder, - self.hs.hostname, - self.hs.config.signing_key[0], - ) - - new_event = builder.build() - + new_event = self._sign_event(event) # Try the host we successfully got a response to /make_join/ # request first. try: @@ -632,11 +601,7 @@ class FederationHandler(BaseHandler): target_hosts.insert(0, origin) except ValueError: pass - - ret = yield self.replication_layer.send_join( - target_hosts, - new_event - ) + ret = yield self.replication_layer.send_join(target_hosts, new_event) origin = ret["origin"] state = ret["state"] @@ -700,7 +665,7 @@ class FederationHandler(BaseHandler): @log_function def on_make_join_request(self, room_id, user_id, query): """ We've received a /make_join/ request, so we create a partial - join event for the room and return that. We don *not* persist or + join event for the room and return that. We do *not* persist or process it until the other server has signed it and sent it back. """ event_content = {"membership": Membership.JOIN} @@ -859,6 +824,168 @@ class FederationHandler(BaseHandler): defer.returnValue(event) + @defer.inlineCallbacks + def do_remotely_reject_invite(self, target_hosts, room_id, user_id): + origin, event = yield self._make_and_verify_event( + target_hosts, + room_id, + user_id, + "leave", + {} + ) + signed_event = self._sign_event(event) + + # Try the host we successfully got a response to /make_join/ + # request first. + try: + target_hosts.remove(origin) + target_hosts.insert(0, origin) + except ValueError: + pass + + yield self.replication_layer.send_leave( + target_hosts, + signed_event + ) + defer.returnValue(None) + + @defer.inlineCallbacks + def _make_and_verify_event(self, target_hosts, room_id, user_id, membership, content): + origin, pdu = yield self.replication_layer.make_membership_event( + target_hosts, + room_id, + user_id, + membership, + content + ) + + logger.debug("Got response to make_%s: %s", membership, pdu) + + event = pdu + + # We should assert some things. + # FIXME: Do this in a nicer way + assert(event.type == EventTypes.Member) + assert(event.user_id == user_id) + assert(event.state_key == user_id) + assert(event.room_id == room_id) + defer.returnValue((origin, event)) + + def _sign_event(self, event): + event.internal_metadata.outlier = False + + builder = self.event_builder_factory.new( + unfreeze(event.get_pdu_json()) + ) + + builder.event_id = self.event_builder_factory.create_event_id() + builder.origin = self.hs.hostname + + if not hasattr(event, "signatures"): + builder.signatures = {} + + add_hashes_and_signatures( + builder, + self.hs.hostname, + self.hs.config.signing_key[0], + ) + + return builder.build() + + @defer.inlineCallbacks + @log_function + def on_make_leave_request(self, room_id, user_id): + """ We've received a /make_leave/ request, so we create a partial + join event for the room and return that. We do *not* persist or + process it until the other server has signed it and sent it back. + """ + builder = self.event_builder_factory.new({ + "type": EventTypes.Member, + "content": {"membership": Membership.LEAVE}, + "room_id": room_id, + "sender": user_id, + "state_key": user_id, + }) + + event, context = yield self._create_new_client_event( + builder=builder, + ) + + self.auth.check(event, auth_events=context.current_state) + + defer.returnValue(event) + + @defer.inlineCallbacks + @log_function + def on_send_leave_request(self, origin, pdu): + """ We have received a leave event for a room. Fully process it.""" + event = pdu + + logger.debug( + "on_send_leave_request: Got event: %s, signatures: %s", + event.event_id, + event.signatures, + ) + + event.internal_metadata.outlier = False + + context, event_stream_id, max_stream_id = yield self._handle_new_event( + origin, event + ) + + logger.debug( + "on_send_leave_request: After _handle_new_event: %s, sigs: %s", + event.event_id, + event.signatures, + ) + + extra_users = [] + if event.type == EventTypes.Member: + target_user_id = event.state_key + target_user = UserID.from_string(target_user_id) + extra_users.append(target_user) + + with PreserveLoggingContext(): + d = self.notifier.on_new_room_event( + event, event_stream_id, max_stream_id, extra_users=extra_users + ) + + def log_failure(f): + logger.warn( + "Failed to notify about %s: %s", + event.event_id, f.value + ) + + d.addErrback(log_failure) + + new_pdu = event + + destinations = set() + + for k, s in context.current_state.items(): + try: + if k[0] == EventTypes.Member: + if s.content["membership"] == Membership.LEAVE: + destinations.add( + UserID.from_string(s.state_key).domain + ) + except: + logger.warn( + "Failed to get destination from event %s", s.event_id + ) + + destinations.discard(origin) + + logger.debug( + "on_send_leave_request: Sending event: %s, signatures: %s", + event.event_id, + event.signatures, + ) + + self.replication_layer.send_pdu(new_pdu, destinations) + + defer.returnValue(None) + @defer.inlineCallbacks def get_state_for_pdu(self, origin, room_id, event_id, do_auth=True): yield run_on_reactor() diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 3f0cde56f0..60f9fa58b0 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -389,7 +389,22 @@ class RoomMemberHandler(BaseHandler): if event.membership == Membership.JOIN: yield self._do_join(event, context, do_auth=do_auth) else: - # This is not a JOIN, so we can handle it normally. + if event.membership == Membership.LEAVE: + is_host_in_room = yield self.is_host_in_room(room_id, context) + if not is_host_in_room: + # Rejecting an invite, rather than leaving a joined room + handler = self.hs.get_handlers().federation_handler + inviter = yield self.get_inviter(event) + if not inviter: + # return the same error as join_room_alias does + raise SynapseError(404, "No known servers") + yield handler.do_remotely_reject_invite( + [inviter.domain], + room_id, + event.user_id + ) + defer.returnValue({"room_id": room_id}) + return # FIXME: This isn't idempotency. if prev_state and prev_state.membership == event.membership: @@ -413,7 +428,7 @@ class RoomMemberHandler(BaseHandler): defer.returnValue({"room_id": room_id}) @defer.inlineCallbacks - def join_room_alias(self, joinee, room_alias, do_auth=True, content={}): + def join_room_alias(self, joinee, room_alias, content={}): directory_handler = self.hs.get_handlers().directory_handler mapping = yield directory_handler.get_association(room_alias) @@ -447,8 +462,6 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def _do_join(self, event, context, room_hosts=None, do_auth=True): - joinee = UserID.from_string(event.state_key) - # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id # XXX: We don't do an auth check if we are doing an invite @@ -456,48 +469,18 @@ class RoomMemberHandler(BaseHandler): # that we are allowed to join when we decide whether or not we # need to do the invite/join dance. - is_host_in_room = yield self.auth.check_host_in_room( - event.room_id, - self.hs.hostname - ) - if not is_host_in_room: - # is *anyone* in the room? - room_member_keys = [ - v for (k, v) in context.current_state.keys() if ( - k == "m.room.member" - ) - ] - if len(room_member_keys) == 0: - # has the room been created so we can join it? - create_event = context.current_state.get(("m.room.create", "")) - if create_event: - is_host_in_room = True - + is_host_in_room = yield self.is_host_in_room(room_id, context) if is_host_in_room: should_do_dance = False elif room_hosts: # TODO: Shouldn't this be remote_room_host? should_do_dance = True else: - # TODO(markjh): get prev_state from snapshot - prev_state = yield self.store.get_room_member( - joinee.to_string(), room_id - ) - - if prev_state and prev_state.membership == Membership.INVITE: - inviter = UserID.from_string(prev_state.user_id) - - should_do_dance = not self.hs.is_mine(inviter) - room_hosts = [inviter.domain] - elif "third_party_invite" in event.content: - if "sender" in event.content["third_party_invite"]: - inviter = UserID.from_string( - event.content["third_party_invite"]["sender"] - ) - should_do_dance = not self.hs.is_mine(inviter) - room_hosts = [inviter.domain] - else: + inviter = yield self.get_inviter(event) + if not inviter: # return the same error as join_room_alias does raise SynapseError(404, "No known servers") + should_do_dance = not self.hs.is_mine(inviter) + room_hosts = [inviter.domain] if should_do_dance: handler = self.hs.get_handlers().federation_handler @@ -505,8 +488,7 @@ class RoomMemberHandler(BaseHandler): room_hosts, room_id, event.user_id, - event.content, # FIXME To get a non-frozen dict - context + event.content # FIXME To get a non-frozen dict ) else: logger.debug("Doing normal join") @@ -523,6 +505,44 @@ class RoomMemberHandler(BaseHandler): "user_joined_room", user=user, room_id=room_id ) + @defer.inlineCallbacks + def get_inviter(self, event): + # TODO(markjh): get prev_state from snapshot + prev_state = yield self.store.get_room_member( + event.user_id, event.room_id + ) + + if prev_state and prev_state.membership == Membership.INVITE: + defer.returnValue(UserID.from_string(prev_state.user_id)) + return + elif "third_party_invite" in event.content: + if "sender" in event.content["third_party_invite"]: + inviter = UserID.from_string( + event.content["third_party_invite"]["sender"] + ) + defer.returnValue(inviter) + defer.returnValue(None) + + @defer.inlineCallbacks + def is_host_in_room(self, room_id, context): + is_host_in_room = yield self.auth.check_host_in_room( + room_id, + self.hs.hostname + ) + if not is_host_in_room: + # is *anyone* in the room? + room_member_keys = [ + v for (k, v) in context.current_state.keys() if ( + k == "m.room.member" + ) + ] + if len(room_member_keys) == 0: + # has the room been created so we can join it? + create_event = context.current_state.get(("m.room.create", "")) + if create_event: + is_host_in_room = True + defer.returnValue(is_host_in_room) + @defer.inlineCallbacks def get_joined_rooms_for_user(self, user): """Returns a list of roomids that the user has any of the given diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index a2123be81b..93896dd076 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -277,10 +277,10 @@ class RoomPermissionsTestCase(RestTestCase): expect_code=403) # set [invite/join/left] of self, set [invite/join/left] of other, - # expect all 403s + # expect all 404s because room doesn't exist on any server for usr in [self.user_id, self.rmcreator_id]: yield self.join(room=room, user=usr, expect_code=404) - yield self.leave(room=room, user=usr, expect_code=403) + yield self.leave(room=room, user=usr, expect_code=404) @defer.inlineCallbacks def test_membership_private_room_perms(self): -- cgit 1.5.1 From 45cd2b023399dc79a77cf59a356ed1c130d970d2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 20 Oct 2015 15:33:25 +0100 Subject: Refactor api.filtering to have a Filter API --- synapse/api/filtering.py | 153 +++++++++++++---------------------- synapse/rest/client/v2_alpha/sync.py | 4 +- tests/api/test_filtering.py | 57 +++++++------ 3 files changed, 88 insertions(+), 126 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index e79e91e7eb..cd7a465e97 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -24,7 +24,7 @@ class Filtering(object): def get_user_filter(self, user_localpart, filter_id): result = self.store.get_user_filter(user_localpart, filter_id) - result.addCallback(Filter) + result.addCallback(FilterCollection) return result def add_user_filter(self, user_localpart, user_filter): @@ -131,125 +131,82 @@ class Filtering(object): raise SynapseError(400, "Bad bundle_updates: expected bool.") -class Filter(object): +class FilterCollection(object): def __init__(self, filter_json): self.filter_json = filter_json + self.room_timeline_filter = Filter( + self.filter_json.get("room", {}).get("timeline", {}) + ) + + self.room_state_filter = Filter( + self.filter_json.get("room", {}).get("state", {}) + ) + + self.room_ephemeral_filter = Filter( + self.filter_json.get("room", {}).get("ephemeral", {}) + ) + + self.presence_filter = Filter( + self.filter_json.get("presence", {}) + ) + def timeline_limit(self): - return self.filter_json.get("room", {}).get("timeline", {}).get("limit", 10) + return self.room_timeline_filter.limit() def presence_limit(self): - return self.filter_json.get("presence", {}).get("limit", 10) + return self.presence_filter.limit() def ephemeral_limit(self): - return self.filter_json.get("room", {}).get("ephemeral", {}).get("limit", 10) + return self.room_ephemeral_filter.limit() def filter_presence(self, events): - return self._filter_on_key(events, ["presence"]) + return self.presence_filter.filter(events) def filter_room_state(self, events): - return self._filter_on_key(events, ["room", "state"]) + return self.room_state_filter.filter(events) def filter_room_timeline(self, events): - return self._filter_on_key(events, ["room", "timeline"]) + return self.room_timeline_filter.filter(events) def filter_room_ephemeral(self, events): - return self._filter_on_key(events, ["room", "ephemeral"]) - - def _filter_on_key(self, events, keys): - filter_json = self.filter_json - if not filter_json: - return events - - try: - # extract the right definition from the filter - definition = filter_json - for key in keys: - definition = definition[key] - return self._filter_with_definition(events, definition) - except KeyError: - # return all events if definition isn't specified. - return events - - def _filter_with_definition(self, events, definition): - return [e for e in events if self._passes_definition(definition, e)] - - def _passes_definition(self, definition, event): - """Check if the event passes the filter definition - Args: - definition(dict): The filter definition to check against - event(dict or Event): The event to check - Returns: - True if the event passes the filter in the definition - """ - if type(event) is dict: - room_id = event.get("room_id") - sender = event.get("sender") - event_type = event["type"] - else: - room_id = getattr(event, "room_id", None) - sender = getattr(event, "sender", None) - event_type = event.type - return self._event_passes_definition( - definition, room_id, sender, event_type - ) + return self.room_ephemeral_filter.filter(events) - def _event_passes_definition(self, definition, room_id, sender, - event_type): - """Check if the event passes through the given definition. - Args: - definition(dict): The definition to check against. - room_id(str): The id of the room this event is in or None. - sender(str): The sender of the event - event_type(str): The type of the event. - Returns: - True if the event passes through the filter. - """ - # Algorithm notes: - # For each key in the definition, check the event meets the criteria: - # * For types: Literal match or prefix match (if ends with wildcard) - # * For senders/rooms: Literal match only - # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' - # and 'not_types' then it is treated as only being in 'not_types') - - # room checks - if room_id is not None: - allow_rooms = definition.get("rooms", None) - reject_rooms = definition.get("not_rooms", None) - if reject_rooms and room_id in reject_rooms: - return False - if allow_rooms and room_id not in allow_rooms: - return False +class Filter(object): + def __init__(self, filter_json): + self.filter_json = filter_json - # sender checks - if sender is not None: - allow_senders = definition.get("senders", None) - reject_senders = definition.get("not_senders", None) - if reject_senders and sender in reject_senders: - return False - if allow_senders and sender not in allow_senders: + def check(self, event): + literal_keys = { + "rooms": lambda v: event.room_id == v, + "senders": lambda v: event.sender == v, + "types": lambda v: _matches_wildcard(event.type, v) + } + + for name, match_func in literal_keys.items(): + not_name = "not_%s" % (name,) + disallowed_values = self.filter_json.get(not_name, []) + if any(map(match_func, disallowed_values)): return False - # type checks - if "not_types" in definition: - for def_type in definition["not_types"]: - if self._event_matches_type(event_type, def_type): + allowed_values = self.filter_json.get(name, None) + if allowed_values is not None: + if not any(map(match_func, allowed_values)): return False - if "types" in definition: - included = False - for def_type in definition["types"]: - if self._event_matches_type(event_type, def_type): - included = True - break - if not included: - return False return True - def _event_matches_type(self, event_type, def_type): - if def_type.endswith("*"): - type_prefix = def_type[:-1] - return event_type.startswith(type_prefix) - else: - return event_type == def_type + def filter(self, events): + return filter(self.check, events) + + def limit(self): + return self.filter_json.get("limit", 10) + + +def _matches_wildcard(actual_value, filter_value): + if filter_value.endswith("*"): + type_prefix = filter_value[:-1] + return actual_value.startswith(type_prefix) + else: + return actual_value == filter_value diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index fffecb24f5..5e27a859f9 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -23,7 +23,7 @@ from synapse.types import StreamToken from synapse.events.utils import ( serialize_event, format_event_for_client_v2_without_event_id, ) -from synapse.api.filtering import Filter +from synapse.api.filtering import FilterCollection from ._base import client_v2_pattern import copy @@ -103,7 +103,7 @@ class SyncRestServlet(RestServlet): user.localpart, filter_id ) except: - filter = Filter({}) + filter = FilterCollection({}) sync_config = SyncConfig( user=user, diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 6942cdac51..9f9af2d783 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -23,10 +23,17 @@ from tests.utils import ( ) from synapse.types import UserID -from synapse.api.filtering import Filter +from synapse.api.filtering import FilterCollection, Filter user_localpart = "test_user" -MockEvent = namedtuple("MockEvent", "sender type room_id") +# MockEvent = namedtuple("MockEvent", "sender type room_id") + + +def MockEvent(**kwargs): + ev = NonCallableMock(spec_set=kwargs.keys()) + ev.configure_mock(**kwargs) + return ev + class FilteringTestCase(unittest.TestCase): @@ -44,7 +51,6 @@ class FilteringTestCase(unittest.TestCase): ) self.filtering = hs.get_filtering() - self.filter = Filter({}) self.datastore = hs.get_datastore() @@ -57,8 +63,9 @@ class FilteringTestCase(unittest.TestCase): type="m.room.message", room_id="!foo:bar" ) + self.assertTrue( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_types_works_with_wildcards(self): @@ -71,7 +78,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_types_works_with_unknowns(self): @@ -84,7 +91,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_not_types_works_with_literals(self): @@ -97,7 +104,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_not_types_works_with_wildcards(self): @@ -110,7 +117,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_not_types_works_with_unknowns(self): @@ -123,7 +130,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_not_types_takes_priority_over_types(self): @@ -137,7 +144,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_senders_works_with_literals(self): @@ -150,7 +157,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_senders_works_with_unknowns(self): @@ -163,7 +170,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_not_senders_works_with_literals(self): @@ -176,7 +183,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_not_senders_works_with_unknowns(self): @@ -189,7 +196,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_not_senders_takes_priority_over_senders(self): @@ -203,7 +210,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_rooms_works_with_literals(self): @@ -216,7 +223,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!secretbase:unknown" ) self.assertTrue( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_rooms_works_with_unknowns(self): @@ -229,7 +236,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_not_rooms_works_with_literals(self): @@ -242,7 +249,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_not_rooms_works_with_unknowns(self): @@ -255,7 +262,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertTrue( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_not_rooms_takes_priority_over_rooms(self): @@ -269,7 +276,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!secretbase:unknown" ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_combined_event(self): @@ -287,7 +294,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertTrue( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_combined_event_bad_sender(self): @@ -305,7 +312,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_combined_event_bad_room(self): @@ -323,7 +330,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!piggyshouse:muppets" # nope ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) def test_definition_combined_event_bad_type(self): @@ -341,7 +348,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertFalse( - self.filter._passes_definition(definition, event) + Filter(definition).check(event) ) @defer.inlineCallbacks @@ -359,7 +366,6 @@ class FilteringTestCase(unittest.TestCase): event = MockEvent( sender="@foo:bar", type="m.profile", - room_id="!foo:bar" ) events = [event] @@ -386,7 +392,6 @@ class FilteringTestCase(unittest.TestCase): event = MockEvent( sender="@foo:bar", type="custom.avatar.3d.crazy", - room_id="!foo:bar" ) events = [event] -- cgit 1.5.1 From 87deec824a6a7b90d463b1e09ad799f5e8e2586c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 20 Oct 2015 15:47:42 +0100 Subject: Docstring --- synapse/api/filtering.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index cd7a465e97..60b6648e0d 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -178,6 +178,11 @@ class Filter(object): self.filter_json = filter_json def check(self, event): + """Checks whether the filter matches the given event. + + Returns: + bool: True if the event matches + """ literal_keys = { "rooms": lambda v: event.room_id == v, "senders": lambda v: event.sender == v, -- cgit 1.5.1 From 7be06680edd6d1bde6e73a91b361fa8cb0a7034d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 20 Oct 2015 16:36:20 +0100 Subject: Include typing events in initial v2 sync --- synapse/handlers/sync.py | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ee6b881de1..e22fe553fd 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -145,6 +145,10 @@ class SyncHandler(BaseHandler): """ now_token = yield self.event_sources.get_current_token() + now_token, typing_by_room = yield self.typing_by_room( + sync_config, now_token + ) + presence_stream = self.event_sources.sources["presence"] # TODO (mjark): This looks wrong, shouldn't we be getting the presence # UP to the present rather than after the present? @@ -164,7 +168,7 @@ class SyncHandler(BaseHandler): for event in room_list: if event.membership == Membership.JOIN: room_sync = yield self.initial_sync_for_joined_room( - event.room_id, sync_config, now_token, + event.room_id, sync_config, now_token, typing_by_room ) joined.append(room_sync) elif event.membership == Membership.INVITE: @@ -182,7 +186,8 @@ class SyncHandler(BaseHandler): )) @defer.inlineCallbacks - def initial_sync_for_joined_room(self, room_id, sync_config, now_token): + def initial_sync_for_joined_room(self, room_id, sync_config, now_token, + typing_by_room): """Sync a room for a client which is starting without any state Returns: A Deferred JoinedSyncResult. @@ -201,9 +206,28 @@ class SyncHandler(BaseHandler): room_id=room_id, timeline=batch, state=current_state_events, - ephemeral=[], + ephemeral=typing_by_room.get(room_id, []), )) + @defer.inlineCallbacks + def typing_by_room(self, sync_config, now_token, since_token=None): + typing_key = since_token.typing_key if since_token else "0" + + typing_source = self.event_sources.sources["typing"] + typing, typing_key = yield typing_source.get_new_events_for_user( + user=sync_config.user, + from_key=typing_key, + limit=sync_config.filter.ephemeral_limit(), + ) + now_token = now_token.copy_and_replace("typing_key", typing_key) + + typing_by_room = {event["room_id"]: [event] for event in typing} + for event in typing: + event.pop("room_id") + logger.debug("Typing %r", typing_by_room) + + defer.returnValue((now_token, typing_by_room)) + @defer.inlineCallbacks def incremental_sync_with_gap(self, sync_config, since_token): """ Get the incremental delta needed to bring the client up to @@ -221,18 +245,9 @@ class SyncHandler(BaseHandler): ) now_token = now_token.copy_and_replace("presence_key", presence_key) - typing_source = self.event_sources.sources["typing"] - typing, typing_key = yield typing_source.get_new_events_for_user( - user=sync_config.user, - from_key=since_token.typing_key, - limit=sync_config.filter.ephemeral_limit(), + now_token, typing_by_room = yield self.typing_by_room( + sync_config, now_token, since_token ) - now_token = now_token.copy_and_replace("typing_key", typing_key) - - typing_by_room = {event["room_id"]: [event] for event in typing} - for event in typing: - event.pop("room_id") - logger.debug("Typing %r", typing_by_room) rm_handler = self.hs.get_handlers().room_member_handler app_service = yield self.store.get_app_service_by_user_id( -- cgit 1.5.1 From ede07434e069d1b143993a3b492428b69a515856 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 21 Oct 2015 09:42:07 +0100 Subject: Use 403 and message to match handlers/auth --- synapse/rest/client/v1/login.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index b2e4cb8eaa..e71cf7e43e 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -102,9 +102,7 @@ class LoginRestServlet(ClientV1RestServlet): login_submission['medium'], login_submission['address'] ) if not user_id: - raise LoginError( - 401, "Unrecognised address", errcode=Codes.UNAUTHORIZED - ) + raise LoginError(403, "", errcode=Codes.FORBIDDEN) else: user_id = login_submission['user'] -- cgit 1.5.1 From c8baada94a6539cfcd1ec1316892302ae2271f4c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 20 Oct 2015 17:09:53 +0100 Subject: Filter search results --- synapse/handlers/search.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 22808b9c07..473aab53f0 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.constants import Membership +from synapse.api.filtering import Filter from synapse.api.errors import SynapseError from synapse.events.utils import serialize_event @@ -49,9 +50,12 @@ class SearchHandler(BaseHandler): keys = content["search_categories"]["room_events"].get("keys", [ "content.body", "content.name", "content.topic", ]) + filter_dict = content["search_categories"]["room_events"].get("filter", {}) except KeyError: raise SynapseError(400, "Invalid search query") + filtr = Filter(filter_dict) + # TODO: Search through left rooms too rooms = yield self.store.get_rooms_for_user_where_membership_is( user.to_string(), @@ -64,11 +68,12 @@ class SearchHandler(BaseHandler): rank_map, event_map = yield self.store.search_msgs(room_ids, search_term, keys) + filtered_events = filtr.filter(event_map.values()) + allowed_events = yield self._filter_events_for_client( - user.to_string(), event_map.values() + user.to_string(), filtered_events ) - # TODO: Filter allowed_events # TODO: Add a limit time_now = self.clock.time_msec() -- cgit 1.5.1 From 5c41224a89a9ceedeb5db10f972c10344382faf2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Oct 2015 10:09:26 +0100 Subject: Filter room ids before hitting the database --- synapse/api/filtering.py | 20 ++++++++++++++++++++ synapse/handlers/search.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 60b6648e0d..ab14b47281 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -202,6 +202,26 @@ class Filter(object): return True + def filter_rooms(self, room_ids): + """Apply the 'rooms' filter to a given list of rooms. + + Args: + room_ids (list): A list of room_ids. + + Returns: + list: A list of room_ids that match the filter + """ + room_ids = set(room_ids) + + disallowed_rooms = set(self.filter_json.get("not_rooms", [])) + room_ids -= disallowed_rooms + + allowed_rooms = self.filter_json.get("rooms", None) + if allowed_rooms is not None: + room_ids &= set(allowed_rooms) + + return room_ids + def filter(self, events): return filter(self.check, events) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 473aab53f0..f53e5d35ac 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -64,7 +64,7 @@ class SearchHandler(BaseHandler): ) room_ids = set(r.room_id for r in rooms) - # TODO: Apply room filter to rooms list + room_ids = filtr.filter_rooms(room_ids) rank_map, event_map = yield self.store.search_msgs(room_ids, search_term, keys) -- cgit 1.5.1 From 4dec901c76e29ad029f53ce199450d75ec8d2ad5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 21 Oct 2015 10:10:55 +0100 Subject: Cap the time to retry txns to appservices to 8.5 minutes There's been numerous issues with people playing around with their application service and then not receiving events from their HS for ages due to backoff timers reaching crazy heights (albeit capped at < 1 day). Reduce the max time between pokes to be 8.5 minutes (2^9 secs) which is quick enough for people to wait it out (avg wait time being 4.25 min) but long enough to actually give the AS breathing room if it needs it. --- synapse/appservice/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 59b0b1f4ac..44dc2c4744 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -224,8 +224,8 @@ class _Recoverer(object): self.clock.call_later((2 ** self.backoff_counter), self.retry) def _backoff(self): - # cap the backoff to be around 18h => (2^16) = 65536 secs - if self.backoff_counter < 16: + # cap the backoff to be around 8.5min => (2^9) = 512 secs + if self.backoff_counter < 9: self.backoff_counter += 1 self.recover() -- cgit 1.5.1 From e3d75f564ab1d17eb4ee47314f78c41553a486f1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 21 Oct 2015 11:15:48 +0100 Subject: Include banned rooms in the archived section of v2 sync --- synapse/handlers/sync.py | 15 +++++++++------ synapse/storage/roommember.py | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5ca2606443..6cb756e476 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -175,9 +175,12 @@ class SyncHandler(BaseHandler): ) room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=sync_config.user.to_string(), - membership_list=[ - Membership.INVITE, Membership.JOIN, Membership.LEAVE - ] + membership_list=( + Membership.INVITE, + Membership.JOIN, + Membership.LEAVE, + Membership.BAN + ) ) joined = [] @@ -195,7 +198,7 @@ class SyncHandler(BaseHandler): room_id=event.room_id, invite=invite, )) - elif event.membership == Membership.LEAVE: + elif event.membership in (Membership.LEAVE, Membership.BAN): leave_token = now_token.copy_and_replace( "room_key", "s%d" % (event.stream_ordering,) ) @@ -327,7 +330,7 @@ class SyncHandler(BaseHandler): and event.state_key == sync_config.user.to_string()): if event.membership == Membership.INVITE: invite_events.append(event) - elif event.membership == Membership.LEAVE: + elif event.membership in (Membership.LEAVE, Membership.BAN): leave_events.append(event) for room_id in joined_room_ids: @@ -362,7 +365,7 @@ class SyncHandler(BaseHandler): sync_config.user.to_string() ) - leave_events = yield self.store.get_leave_events_for_user( + leave_events = yield self.store.get_leave_and_ban_events_for_user( sync_config.user.to_string() ) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 623400fd36..ae1ad56d9a 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -124,7 +124,7 @@ class RoomMemberStore(SQLBaseStore): invites.event_id for invite in invites ])) - def get_leave_events_for_user(self, user_id): + def get_leave_and_ban_events_for_user(self, user_id): """ Get all the leave events for a user Args: user_id (str): The user ID. @@ -132,7 +132,7 @@ class RoomMemberStore(SQLBaseStore): A deferred list of event objects. """ return self.get_rooms_for_user_where_membership_is( - user_id, [Membership.LEAVE] + user_id, (Membership.LEAVE, Membership.BAN) ).addCallback(lambda leaves: self._get_events([ leave.event_id for leave in leaves ])) -- cgit 1.5.1 From d63a0ca34b0951cb3d1981225e1c1cf91b996d30 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 21 Oct 2015 15:45:37 +0100 Subject: Doc string for the SyncHandler.typing_by_room method --- synapse/handlers/sync.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index e22fe553fd..e651b49987 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -211,6 +211,18 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def typing_by_room(self, sync_config, now_token, since_token=None): + """Get the typing events for each room the user is in + Args: + sync_config (SyncConfig): The flags, filters and user for the sync. + now_token (StreamToken): Where the server is currently up to. + since_token (StreamToken): Where the server was when the client + last synced. + Returns: + A tuple of the now StreamToken, updated to reflect the which typing + events are included, and a dict mapping from room_id to a list of + typing events for that room. + """ + typing_key = since_token.typing_key if since_token else "0" typing_source = self.event_sources.sources["typing"] -- cgit 1.5.1 From 5025ba959f2b91919a13d1c3b014487d68c41ad7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Oct 2015 10:37:04 +0100 Subject: Add config option to disable password login --- synapse/config/cas.py | 3 ++- synapse/config/homeserver.py | 4 +++- synapse/config/password.py | 32 ++++++++++++++++++++++++++++++++ synapse/config/saml2.py | 3 ++- synapse/rest/client/v1/login.py | 8 +++++++- 5 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 synapse/config/password.py diff --git a/synapse/config/cas.py b/synapse/config/cas.py index d268680729..a337ae6ca0 100644 --- a/synapse/config/cas.py +++ b/synapse/config/cas.py @@ -25,7 +25,7 @@ class CasConfig(Config): def read_config(self, config): cas_config = config.get("cas_config", None) if cas_config: - self.cas_enabled = True + self.cas_enabled = cas_config.get("enabled", True) self.cas_server_url = cas_config["server_url"] self.cas_required_attributes = cas_config.get("required_attributes", {}) else: @@ -37,6 +37,7 @@ class CasConfig(Config): return """ # Enable CAS for registration and login. #cas_config: + # enabled: true # server_url: "https://cas-server.com" # #required_attributes: # # name: value diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 3039f3c0bf..4743e6abc5 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -27,12 +27,14 @@ from .appservice import AppServiceConfig from .key import KeyConfig from .saml2 import SAML2Config from .cas import CasConfig +from .password import PasswordConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, MetricsConfig, - AppServiceConfig, KeyConfig, SAML2Config, CasConfig): + AppServiceConfig, KeyConfig, SAML2Config, CasConfig, + PasswordConfig,): pass diff --git a/synapse/config/password.py b/synapse/config/password.py new file mode 100644 index 0000000000..1a3e278472 --- /dev/null +++ b/synapse/config/password.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class PasswordConfig(Config): + """Password login configuration + """ + + def read_config(self, config): + password_config = config.get("password_config", {}) + self.password_enabled = password_config.get("enabled", True) + + def default_config(self, config_dir_path, server_name, **kwargs): + return """ + # Enable password for login. + password_config: + enabled: true + """ diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py index 4c6133cf22..8d7f443021 100644 --- a/synapse/config/saml2.py +++ b/synapse/config/saml2.py @@ -33,7 +33,7 @@ class SAML2Config(Config): def read_config(self, config): saml2_config = config.get("saml2_config", None) if saml2_config: - self.saml2_enabled = True + self.saml2_enabled = saml2_config.get("enabled", True) self.saml2_config_path = saml2_config["config_path"] self.saml2_idp_redirect_url = saml2_config["idp_redirect_url"] else: @@ -49,6 +49,7 @@ class SAML2Config(Config): # the user back to /login/saml2 with proper info. # See pysaml2 docs for format of config. #saml2_config: + # enabled: true # config_path: "%s/sp_conf.py" # idp_redirect_url: "http://%s/idp" """ % (config_dir_path, server_name) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 2e3e4f39f3..00ec8fcd74 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -43,6 +43,7 @@ class LoginRestServlet(ClientV1RestServlet): def __init__(self, hs): super(LoginRestServlet, self).__init__(hs) self.idp_redirect_url = hs.config.saml2_idp_redirect_url + self.password_enabled = hs.config.password_enabled self.saml2_enabled = hs.config.saml2_enabled self.cas_enabled = hs.config.cas_enabled self.cas_server_url = hs.config.cas_server_url @@ -50,11 +51,13 @@ class LoginRestServlet(ClientV1RestServlet): self.servername = hs.config.server_name def on_GET(self, request): - flows = [{"type": LoginRestServlet.PASS_TYPE}] + flows = [] if self.saml2_enabled: flows.append({"type": LoginRestServlet.SAML2_TYPE}) if self.cas_enabled: flows.append({"type": LoginRestServlet.CAS_TYPE}) + if self.password_enabled: + flows.append({"type": LoginRestServlet.PASS_TYPE}) return (200, {"flows": flows}) def on_OPTIONS(self, request): @@ -65,6 +68,9 @@ class LoginRestServlet(ClientV1RestServlet): login_submission = _parse_json(request) try: if login_submission["type"] == LoginRestServlet.PASS_TYPE: + if not self.password_enabled: + raise SynapseError(400, "Password login has been disabled.") + result = yield self.do_password_login(login_submission) defer.returnValue(result) elif self.saml2_enabled and (login_submission["type"] == -- cgit 1.5.1 From 3ce1b8c70539795dfe060b3ea3e211181a730ed3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Oct 2015 10:43:35 +0100 Subject: Don't keep appending report_stats to demo config --- demo/start.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index d90115ec97..dcc4d6f4fa 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -38,8 +38,12 @@ for port in 8080 8081 8082; do perl -p -i -e 's/^enable_registration:.*/enable_registration: true/g' $DIR/etc/$port.config - echo "full_twisted_stacktraces: true" >> $DIR/etc/$port.config - echo "report_stats: false" >> $DIR/etc/$port.config + if ! grep -F "full_twisted_stacktraces" -q $DIR/etc/$port.config; then + echo "full_twisted_stacktraces: true" >> $DIR/etc/$port.config + fi + if ! grep -F "report_stats" -q $DIR/etc/$port.config ; then + echo "report_stats: false" >> $DIR/etc/$port.config + fi python -m synapse.app.homeserver \ --config-path "$DIR/etc/$port.config" \ -- cgit 1.5.1 From 4d25bc6c92c3d219786892c5d510e0c4c4eb8b96 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Oct 2015 11:12:28 +0100 Subject: Move FTS to delta 25 --- synapse/storage/prepare_database.py | 2 +- synapse/storage/schema/delta/24/fts.py | 123 --------------------------------- synapse/storage/schema/delta/25/fts.py | 123 +++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 124 deletions(-) delete mode 100644 synapse/storage/schema/delta/24/fts.py create mode 100644 synapse/storage/schema/delta/25/fts.py diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 1ddf55be4d..1a74d6e360 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 24 +SCHEMA_VERSION = 25 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/24/fts.py b/synapse/storage/schema/delta/24/fts.py deleted file mode 100644 index 0c752d8426..0000000000 --- a/synapse/storage/schema/delta/24/fts.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from synapse.storage.prepare_database import get_statements -from synapse.storage.engines import PostgresEngine, Sqlite3Engine - -import ujson - -logger = logging.getLogger(__name__) - - -POSTGRES_SQL = """ -CREATE TABLE event_search ( - event_id TEXT, - room_id TEXT, - key TEXT, - vector tsvector -); - -INSERT INTO event_search SELECT - event_id, room_id, 'content.body', - to_tsvector('english', json::json->'content'->>'body') - FROM events NATURAL JOIN event_json WHERE type = 'm.room.message'; - -INSERT INTO event_search SELECT - event_id, room_id, 'content.name', - to_tsvector('english', json::json->'content'->>'name') - FROM events NATURAL JOIN event_json WHERE type = 'm.room.name'; - -INSERT INTO event_search SELECT - event_id, room_id, 'content.topic', - to_tsvector('english', json::json->'content'->>'topic') - FROM events NATURAL JOIN event_json WHERE type = 'm.room.topic'; - - -CREATE INDEX event_search_fts_idx ON event_search USING gin(vector); -CREATE INDEX event_search_ev_idx ON event_search(event_id); -CREATE INDEX event_search_ev_ridx ON event_search(room_id); -""" - - -SQLITE_TABLE = ( - "CREATE VIRTUAL TABLE event_search USING fts3 ( event_id, room_id, key, value)" -) - - -def run_upgrade(cur, database_engine, *args, **kwargs): - if isinstance(database_engine, PostgresEngine): - run_postgres_upgrade(cur) - return - - if isinstance(database_engine, Sqlite3Engine): - run_sqlite_upgrade(cur) - return - - -def run_postgres_upgrade(cur): - for statement in get_statements(POSTGRES_SQL.splitlines()): - cur.execute(statement) - - -def run_sqlite_upgrade(cur): - cur.execute(SQLITE_TABLE) - - rowid = -1 - while True: - cur.execute( - "SELECT rowid, json FROM event_json" - " WHERE rowid > ?" - " ORDER BY rowid ASC LIMIT 100", - (rowid,) - ) - - res = cur.fetchall() - - if not res: - break - - events = [ - ujson.loads(js) - for _, js in res - ] - - rowid = max(rid for rid, _ in res) - - rows = [] - for ev in events: - if ev["type"] == "m.room.message": - rows.append(( - ev["event_id"], ev["room_id"], "content.body", - ev["content"]["body"] - )) - if ev["type"] == "m.room.name": - rows.append(( - ev["event_id"], ev["room_id"], "content.name", - ev["content"]["name"] - )) - if ev["type"] == "m.room.topic": - rows.append(( - ev["event_id"], ev["room_id"], "content.topic", - ev["content"]["topic"] - )) - - if rows: - logger.info(rows) - cur.executemany( - "INSERT INTO event_search (event_id, room_id, key, value)" - " VALUES (?,?,?,?)", - rows - ) diff --git a/synapse/storage/schema/delta/25/fts.py b/synapse/storage/schema/delta/25/fts.py new file mode 100644 index 0000000000..fd181fc630 --- /dev/null +++ b/synapse/storage/schema/delta/25/fts.py @@ -0,0 +1,123 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from synapse.storage.prepare_database import get_statements +from synapse.storage.engines import PostgresEngine, Sqlite3Engine + +import ujson + +logger = logging.getLogger(__name__) + + +POSTGRES_SQL = """ +CREATE TABLE IF NOT EXISTS event_search ( + event_id TEXT, + room_id TEXT, + key TEXT, + vector tsvector +); + +INSERT INTO event_search SELECT + event_id, room_id, 'content.body', + to_tsvector('english', json::json->'content'->>'body') + FROM events NATURAL JOIN event_json WHERE type = 'm.room.message'; + +INSERT INTO event_search SELECT + event_id, room_id, 'content.name', + to_tsvector('english', json::json->'content'->>'name') + FROM events NATURAL JOIN event_json WHERE type = 'm.room.name'; + +INSERT INTO event_search SELECT + event_id, room_id, 'content.topic', + to_tsvector('english', json::json->'content'->>'topic') + FROM events NATURAL JOIN event_json WHERE type = 'm.room.topic'; + + +CREATE INDEX event_search_fts_idx ON event_search USING gin(vector); +CREATE INDEX event_search_ev_idx ON event_search(event_id); +CREATE INDEX event_search_ev_ridx ON event_search(room_id); +""" + + +SQLITE_TABLE = ( + "CREATE VIRTUAL TABLE IF NOT EXISTS event_search USING fts3 ( event_id, room_id, key, value)" +) + + +def run_upgrade(cur, database_engine, *args, **kwargs): + if isinstance(database_engine, PostgresEngine): + run_postgres_upgrade(cur) + return + + if isinstance(database_engine, Sqlite3Engine): + run_sqlite_upgrade(cur) + return + + +def run_postgres_upgrade(cur): + for statement in get_statements(POSTGRES_SQL.splitlines()): + cur.execute(statement) + + +def run_sqlite_upgrade(cur): + cur.execute(SQLITE_TABLE) + + rowid = -1 + while True: + cur.execute( + "SELECT rowid, json FROM event_json" + " WHERE rowid > ?" + " ORDER BY rowid ASC LIMIT 100", + (rowid,) + ) + + res = cur.fetchall() + + if not res: + break + + events = [ + ujson.loads(js) + for _, js in res + ] + + rowid = max(rid for rid, _ in res) + + rows = [] + for ev in events: + if ev["type"] == "m.room.message": + rows.append(( + ev["event_id"], ev["room_id"], "content.body", + ev["content"]["body"] + )) + if ev["type"] == "m.room.name": + rows.append(( + ev["event_id"], ev["room_id"], "content.name", + ev["content"]["name"] + )) + if ev["type"] == "m.room.topic": + rows.append(( + ev["event_id"], ev["room_id"], "content.topic", + ev["content"]["topic"] + )) + + if rows: + logger.info(rows) + cur.executemany( + "INSERT INTO event_search (event_id, room_id, key, value)" + " VALUES (?,?,?,?)", + rows + ) -- cgit 1.5.1 From f142898f529f89c5bd90710db2d0771894b0b33f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Oct 2015 11:18:01 +0100 Subject: PEP8 --- synapse/storage/schema/delta/25/fts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/25/fts.py b/synapse/storage/schema/delta/25/fts.py index fd181fc630..ed3cc06557 100644 --- a/synapse/storage/schema/delta/25/fts.py +++ b/synapse/storage/schema/delta/25/fts.py @@ -53,7 +53,8 @@ CREATE INDEX event_search_ev_ridx ON event_search(room_id); SQLITE_TABLE = ( - "CREATE VIRTUAL TABLE IF NOT EXISTS event_search USING fts3 ( event_id, room_id, key, value)" + "CREATE VIRTUAL TABLE IF NOT EXISTS event_search" + " USING fts3 ( event_id, room_id, key, value)" ) -- cgit 1.5.1 From e60dad86ba8528d81ffcd1123bf8aa019110bb5d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 22 Oct 2015 11:44:31 +0100 Subject: Reject events which are too large SPEC-222 --- synapse/api/auth.py | 22 +++++++++++++++++++++- synapse/api/errors.py | 9 +++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 494c8ac3d4..88445fe999 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -14,13 +14,14 @@ # limitations under the License. """This module contains classes for authenticating the user.""" +from canonicaljson import encode_canonical_json from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json, SignatureVerifyException from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules -from synapse.api.errors import AuthError, Codes, SynapseError +from synapse.api.errors import AuthError, Codes, SynapseError, EventSizeError from synapse.types import RoomID, UserID, EventID from synapse.util.logutils import log_function from synapse.util import third_party_invites @@ -64,6 +65,8 @@ class Auth(object): Returns: True if the auth checks pass. """ + self.check_size_limits(event) + try: if not hasattr(event, "room_id"): raise AuthError(500, "Event has no room_id: %s" % event) @@ -131,6 +134,23 @@ class Auth(object): logger.info("Denying! %s", event) raise + def check_size_limits(self, event): + def too_big(field): + raise EventSizeError("%s too large" % (field,)) + + if len(event.user_id) > 255: + too_big("user_id") + if len(event.room_id) > 255: + too_big("room_id") + if event.is_state() and len(event.state_key) > 255: + too_big("state_key") + if len(event.type) > 255: + too_big("type") + if len(event.event_id) > 255: + too_big("event_id") + if len(encode_canonical_json(event.get_pdu_json())) > 65536: + too_big("event") + @defer.inlineCallbacks def check_joined_room(self, room_id, user_id, current_state=None): """Check if the user is currently joined in the room diff --git a/synapse/api/errors.py b/synapse/api/errors.py index d1356eb4d9..b3fea27d0e 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -119,6 +119,15 @@ class AuthError(SynapseError): super(AuthError, self).__init__(*args, **kwargs) +class EventSizeError(SynapseError): + """An error raised when an event is too big.""" + + def __init__(self, *args, **kwargs): + if "errcode" not in kwargs: + kwargs["errcode"] = Codes.TOO_LARGE + super(EventSizeError, self).__init__(413, *args, **kwargs) + + class EventStreamError(SynapseError): """An error raised when there a problem with the event stream.""" def __init__(self, *args, **kwargs): -- cgit 1.5.1 From ba02bba88c6895bc9196d226a2bbc8047b93e28b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Oct 2015 13:25:22 +0100 Subject: Limit max number of SQL vars --- synapse/storage/search.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index a3c69c5ab3..810b5406ad 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -36,10 +36,12 @@ class SearchStore(SQLBaseStore): clauses = [] args = [] - clauses.append( - "room_id IN (%s)" % (",".join(["?"] * len(room_ids)),) - ) - args.extend(room_ids) + # Make sure we don't explode because the person is in too many rooms. + if len(room_ids) > 500: + clauses.append( + "room_id IN (%s)" % (",".join(["?"] * len(room_ids)),) + ) + args.extend(room_ids) local_clauses = [] for key in keys: -- cgit 1.5.1 From 232beb3a3c28ccdc5388daa9396d5054b7768b12 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Oct 2015 15:02:35 +0100 Subject: Use namedtuple as return value --- synapse/handlers/search.py | 4 +++- synapse/storage/search.py | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index f53e5d35ac..bdc79ffc55 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -66,7 +66,9 @@ class SearchHandler(BaseHandler): room_ids = filtr.filter_rooms(room_ids) - rank_map, event_map = yield self.store.search_msgs(room_ids, search_term, keys) + rank_map, event_map, _ = yield self.store.search_msgs( + room_ids, search_term, keys + ) filtered_events = filtr.filter(event_map.values()) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 810b5406ad..41451ade57 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -18,6 +18,17 @@ from twisted.internet import defer from _base import SQLBaseStore from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from collections import namedtuple + +"""The result of a search. + +Fields: + rank_map (dict): Mapping event_id -> rank + event_map (dict): Mapping event_id -> event + pagination_token (str): Pagination token +""" +SearchResult = namedtuple("SearchResult", ("rank_map", "event_map", "pagination_token")) + class SearchStore(SQLBaseStore): @defer.inlineCallbacks @@ -31,7 +42,7 @@ class SearchStore(SQLBaseStore): "content.body", "content.name", "content.topic" Returns: - 2-tuple of (dict event_id -> rank, dict event_id -> event) + SearchResult """ clauses = [] args = [] @@ -85,11 +96,12 @@ class SearchStore(SQLBaseStore): for ev in events } - defer.returnValue(( + defer.returnValue(SearchResult( { r["event_id"]: r["rank"] for r in results if r["event_id"] in event_map }, - event_map + event_map, + None )) -- cgit 1.5.1 From 61547106f51e5709c8deba83fd6748b71480c4d5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Oct 2015 16:17:12 +0100 Subject: Fix receipts for room initial sync --- synapse/handlers/receipts.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 86c911c4bf..a47ae3df42 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -156,13 +156,7 @@ class ReceiptsHandler(BaseHandler): if not result: defer.returnValue([]) - event = { - "type": "m.receipt", - "room_id": room_id, - "content": result, - } - - defer.returnValue([event]) + defer.returnValue(result) class ReceiptEventSource(object): -- cgit 1.5.1 From fb0fecd0b9f430670ca6de1f4746601dd6cc24f8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Oct 2015 16:18:35 +0100 Subject: LESS THAN --- synapse/storage/search.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 41451ade57..e658e07dc7 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -48,7 +48,8 @@ class SearchStore(SQLBaseStore): args = [] # Make sure we don't explode because the person is in too many rooms. - if len(room_ids) > 500: + # We filter the results regardless. + if len(room_ids) < 500: clauses.append( "room_id IN (%s)" % (",".join(["?"] * len(room_ids)),) ) -- cgit 1.5.1 From 2980136d7535077f0513b8a12fd7f224700ca140 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Oct 2015 16:19:53 +0100 Subject: Rename --- synapse/handlers/search.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index bdc79ffc55..bbe82b1425 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -54,7 +54,7 @@ class SearchHandler(BaseHandler): except KeyError: raise SynapseError(400, "Invalid search query") - filtr = Filter(filter_dict) + search_filter = Filter(filter_dict) # TODO: Search through left rooms too rooms = yield self.store.get_rooms_for_user_where_membership_is( @@ -64,13 +64,13 @@ class SearchHandler(BaseHandler): ) room_ids = set(r.room_id for r in rooms) - room_ids = filtr.filter_rooms(room_ids) + room_ids = search_filter.filter_rooms(room_ids) rank_map, event_map, _ = yield self.store.search_msgs( room_ids, search_term, keys ) - filtered_events = filtr.filter(event_map.values()) + filtered_events = search_filter.filter(event_map.values()) allowed_events = yield self._filter_events_for_client( user.to_string(), filtered_events -- cgit 1.5.1 From 9b6f3bc7423008fa2d66d88227675b5c1c11db48 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 22 Oct 2015 16:38:03 +0100 Subject: Support filtering events represented as dicts. This is useful because the emphemeral events such as presence and typing are represented as dicts inside synapse. --- synapse/api/filtering.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 60b6648e0d..522b151c35 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -183,10 +183,29 @@ class Filter(object): Returns: bool: True if the event matches """ + if isinstance(event, dict): + return self.check_fields( + event.get("room_id", None), + event.get("sender", None), + event.get("type", None), + ) + else: + return self.check_fields( + event.room_id, + event.sender, + event.type, + ) + + def check_fields(self, room_id, sender, event_type): + """Checks whether the filter matches the given event fields. + + Returns: + bool: True if the event fields match + """ literal_keys = { - "rooms": lambda v: event.room_id == v, - "senders": lambda v: event.sender == v, - "types": lambda v: _matches_wildcard(event.type, v) + "rooms": lambda v: room_id == v, + "senders": lambda v: sender == v, + "types": lambda v: _matches_wildcard(event_type, v) } for name, match_func in literal_keys.items(): -- cgit 1.5.1 From 671ac699f1d4672bed3817b3cafb7498df99c030 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Oct 2015 16:54:56 +0100 Subject: Actually filter results --- synapse/storage/search.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index e658e07dc7..9608b5d6a7 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -48,7 +48,7 @@ class SearchStore(SQLBaseStore): args = [] # Make sure we don't explode because the person is in too many rooms. - # We filter the results regardless. + # We filter the results below regardless. if len(room_ids) < 500: clauses.append( "room_id IN (%s)" % (",".join(["?"] * len(room_ids)),) @@ -66,13 +66,13 @@ class SearchStore(SQLBaseStore): if isinstance(self.database_engine, PostgresEngine): sql = ( - "SELECT ts_rank_cd(vector, query) AS rank, event_id" + "SELECT ts_rank_cd(vector, query) AS rank, room_id, event_id" " FROM plainto_tsquery('english', ?) as query, event_search" " WHERE vector @@ query" ) elif isinstance(self.database_engine, Sqlite3Engine): sql = ( - "SELECT 0 as rank, event_id FROM event_search" + "SELECT 0 as rank, room_id, event_id FROM event_search" " WHERE value MATCH ?" ) else: @@ -90,6 +90,8 @@ class SearchStore(SQLBaseStore): "search_msgs", self.cursor_to_dict, sql, *([search_term] + args) ) + results = filter(lambda row: row["room_id"] in room_ids, results) + events = yield self._get_events([r["event_id"] for r in results]) event_map = { -- cgit 1.5.1 From 4e05aab4f7daa79d3a521f3477f6ade10157350b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 22 Oct 2015 17:08:59 +0100 Subject: Don't assume that the event has a room_id or sender --- synapse/api/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 522b151c35..765d1bc9d5 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -191,8 +191,8 @@ class Filter(object): ) else: return self.check_fields( - event.room_id, - event.sender, + getattr(event, "room_id", None), + getattr(event, "sender", None), event.type, ) -- cgit 1.5.1 From 0c36098c1fe561629651e446589a644fed9188e4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 23 Oct 2015 13:23:48 +0100 Subject: Implement rank function for SQLite FTS --- synapse/storage/engines/sqlite3.py | 27 +++++++++++++++++++++++++++ synapse/storage/schema/delta/25/fts.py | 2 +- synapse/storage/search.py | 3 ++- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index bad3b5c5ac..a5a54ec011 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -17,6 +17,8 @@ from synapse.storage.prepare_database import ( prepare_database, prepare_sqlite3_database ) +import struct + class Sqlite3Engine(object): single_threaded = True @@ -32,6 +34,7 @@ class Sqlite3Engine(object): def on_new_connection(self, db_conn): self.prepare_database(db_conn) + db_conn.create_function("rank", 1, _rank) def prepare_database(self, db_conn): prepare_sqlite3_database(db_conn) @@ -45,3 +48,27 @@ class Sqlite3Engine(object): def lock_table(self, txn, table): return + + +# Following functions taken from: https://github.com/coleifer/peewee + +def _parse_match_info(buf): + bufsize = len(buf) + return [struct.unpack('@I', buf[i:i+4])[0] for i in range(0, bufsize, 4)] + + +def _rank(raw_match_info): + """Handle match_info called w/default args 'pcx' - based on the example rank + function http://sqlite.org/fts3.html#appendix_a + """ + match_info = _parse_match_info(raw_match_info) + score = 0.0 + p, c = match_info[:2] + for phrase_num in range(p): + phrase_info_idx = 2 + (phrase_num * c * 3) + for col_num in range(c): + col_idx = phrase_info_idx + (col_num * 3) + x1, x2 = match_info[col_idx:col_idx + 2] + if x1 > 0: + score += float(x1) / x2 + return score diff --git a/synapse/storage/schema/delta/25/fts.py b/synapse/storage/schema/delta/25/fts.py index ed3cc06557..3f0b02d119 100644 --- a/synapse/storage/schema/delta/25/fts.py +++ b/synapse/storage/schema/delta/25/fts.py @@ -54,7 +54,7 @@ CREATE INDEX event_search_ev_ridx ON event_search(room_id); SQLITE_TABLE = ( "CREATE VIRTUAL TABLE IF NOT EXISTS event_search" - " USING fts3 ( event_id, room_id, key, value)" + " USING fts4 ( event_id, room_id, key, value )" ) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 9608b5d6a7..cdf003502f 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -72,7 +72,8 @@ class SearchStore(SQLBaseStore): ) elif isinstance(self.database_engine, Sqlite3Engine): sql = ( - "SELECT 0 as rank, room_id, event_id FROM event_search" + "SELECT rank(matchinfo(event_search)) as rank, room_id, event_id" + " FROM event_search" " WHERE value MATCH ?" ) else: -- cgit 1.5.1 From 4cf633d5e9628a56cf9b400ee3e073fcdcb365f0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 23 Oct 2015 15:41:36 +0100 Subject: Pull out sender when computing search results --- synapse/storage/schema/delta/25/fts.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/synapse/storage/schema/delta/25/fts.py b/synapse/storage/schema/delta/25/fts.py index 3f0b02d119..056487af30 100644 --- a/synapse/storage/schema/delta/25/fts.py +++ b/synapse/storage/schema/delta/25/fts.py @@ -26,22 +26,23 @@ POSTGRES_SQL = """ CREATE TABLE IF NOT EXISTS event_search ( event_id TEXT, room_id TEXT, + sender TEXT, key TEXT, vector tsvector ); INSERT INTO event_search SELECT - event_id, room_id, 'content.body', + event_id, room_id, json::json->>'sender', 'content.body', to_tsvector('english', json::json->'content'->>'body') FROM events NATURAL JOIN event_json WHERE type = 'm.room.message'; INSERT INTO event_search SELECT - event_id, room_id, 'content.name', + event_id, room_id, json::json->>'sender', 'content.name', to_tsvector('english', json::json->'content'->>'name') FROM events NATURAL JOIN event_json WHERE type = 'm.room.name'; INSERT INTO event_search SELECT - event_id, room_id, 'content.topic', + event_id, room_id, json::json->>'sender', 'content.topic', to_tsvector('english', json::json->'content'->>'topic') FROM events NATURAL JOIN event_json WHERE type = 'm.room.topic'; @@ -99,26 +100,28 @@ def run_sqlite_upgrade(cur): rows = [] for ev in events: - if ev["type"] == "m.room.message": + content = ev.get("content", {}) + body = content.get("body", None) + name = content.get("name", None) + topic = content.get("topic", None) + sender = ev.get("sender", None) + if ev["type"] == "m.room.message" and body: rows.append(( - ev["event_id"], ev["room_id"], "content.body", - ev["content"]["body"] + ev["event_id"], ev["room_id"], sender, "content.body", body )) - if ev["type"] == "m.room.name": + if ev["type"] == "m.room.name" and name: rows.append(( - ev["event_id"], ev["room_id"], "content.name", - ev["content"]["name"] + ev["event_id"], ev["room_id"], sender, "content.name", name )) - if ev["type"] == "m.room.topic": + if ev["type"] == "m.room.topic" and topic: rows.append(( - ev["event_id"], ev["room_id"], "content.topic", - ev["content"]["topic"] + ev["event_id"], ev["room_id"], sender, "content.topic", topic )) if rows: logger.info(rows) cur.executemany( - "INSERT INTO event_search (event_id, room_id, key, value)" - " VALUES (?,?,?,?)", + "INSERT INTO event_search (event_id, room_id, sender, key, value)" + " VALUES (?,?,?,?,?)", rows ) -- cgit 1.5.1 From fa1cf5ef34e684b58433119d78dc95923955ff4f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Oct 2015 15:35:07 +0000 Subject: Move static folder into synapse This is because otherwise it won't get picked up by python packaging. This also fixes the problem where the "static" folder was found if synapse wasn't started from that directory. --- static/client/register/index.html | 32 ---- static/client/register/js/jquery-2.1.3.min.js | 4 - static/client/register/js/recaptcha_ajax.js | 195 --------------------- static/client/register/js/register.js | 117 ------------- static/client/register/register_config.sample.js | 3 - static/client/register/style.css | 60 ------- synapse/app/homeserver.py | 4 +- synapse/static/client/register/index.html | 32 ++++ .../static/client/register/js/jquery-2.1.3.min.js | 4 + .../static/client/register/js/recaptcha_ajax.js | 195 +++++++++++++++++++++ synapse/static/client/register/js/register.js | 117 +++++++++++++ .../client/register/register_config.sample.js | 3 + synapse/static/client/register/style.css | 60 +++++++ 13 files changed, 414 insertions(+), 412 deletions(-) delete mode 100644 static/client/register/index.html delete mode 100644 static/client/register/js/jquery-2.1.3.min.js delete mode 100644 static/client/register/js/recaptcha_ajax.js delete mode 100644 static/client/register/js/register.js delete mode 100644 static/client/register/register_config.sample.js delete mode 100644 static/client/register/style.css create mode 100644 synapse/static/client/register/index.html create mode 100644 synapse/static/client/register/js/jquery-2.1.3.min.js create mode 100644 synapse/static/client/register/js/recaptcha_ajax.js create mode 100644 synapse/static/client/register/js/register.js create mode 100644 synapse/static/client/register/register_config.sample.js create mode 100644 synapse/static/client/register/style.css diff --git a/static/client/register/index.html b/static/client/register/index.html deleted file mode 100644 index 600b3ee41e..0000000000 --- a/static/client/register/index.html +++ /dev/null @@ -1,32 +0,0 @@ - - - Registration - - - - - - - - -
-
- Create account:
- -
- -
- -
- -
- -
-
- - -
-
-
- - diff --git a/static/client/register/js/jquery-2.1.3.min.js b/static/client/register/js/jquery-2.1.3.min.js deleted file mode 100644 index 25714ed29a..0000000000 --- a/static/client/register/js/jquery-2.1.3.min.js +++ /dev/null @@ -1,4 +0,0 @@ -/*! jQuery v2.1.3 | (c) 2005, 2014 jQuery Foundation, Inc. | jquery.org/license */ -!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k={},l=a.document,m="2.1.3",n=function(a,b){return new n.fn.init(a,b)},o=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,p=/^-ms-/,q=/-([\da-z])/gi,r=function(a,b){return b.toUpperCase()};n.fn=n.prototype={jquery:m,constructor:n,selector:"",length:0,toArray:function(){return d.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var b=n.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a,b){return n.each(this,a,b)},map:function(a){return this.pushStack(n.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},n.extend=n.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||n.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(a=arguments[h]))for(b in a)c=g[b],d=a[b],g!==d&&(j&&d&&(n.isPlainObject(d)||(e=n.isArray(d)))?(e?(e=!1,f=c&&n.isArray(c)?c:[]):f=c&&n.isPlainObject(c)?c:{},g[b]=n.extend(j,f,d)):void 0!==d&&(g[b]=d));return g},n.extend({expando:"jQuery"+(m+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===n.type(a)},isArray:Array.isArray,isWindow:function(a){return null!=a&&a===a.window},isNumeric:function(a){return!n.isArray(a)&&a-parseFloat(a)+1>=0},isPlainObject:function(a){return"object"!==n.type(a)||a.nodeType||n.isWindow(a)?!1:a.constructor&&!j.call(a.constructor.prototype,"isPrototypeOf")?!1:!0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?h[i.call(a)]||"object":typeof a},globalEval:function(a){var b,c=eval;a=n.trim(a),a&&(1===a.indexOf("use strict")?(b=l.createElement("script"),b.text=a,l.head.appendChild(b).parentNode.removeChild(b)):c(a))},camelCase:function(a){return a.replace(p,"ms-").replace(q,r)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=s(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(o,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(s(Object(a))?n.merge(c,"string"==typeof a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){return null==b?-1:g.call(b,a,c)},merge:function(a,b){for(var c=+b.length,d=0,e=a.length;c>d;d++)a[e++]=b[d];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,f=0,g=a.length,h=s(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof b&&(c=a[b],b=a,a=c),n.isFunction(a)?(e=d.call(arguments,2),f=function(){return a.apply(b||this,e.concat(d.call(arguments)))},f.guid=a.guid=a.guid||n.guid++,f):void 0},now:Date.now,support:k}),n.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object "+b+"]"]=b.toLowerCase()});function s(a){var b=a.length,c=n.type(a);return"function"===c||n.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var t=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=hb(),z=hb(),A=hb(),B=function(a,b){return a===b&&(l=!0),0},C=1<<31,D={}.hasOwnProperty,E=[],F=E.pop,G=E.push,H=E.push,I=E.slice,J=function(a,b){for(var c=0,d=a.length;d>c;c++)if(a[c]===b)return c;return-1},K="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",L="[\\x20\\t\\r\\n\\f]",M="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",N=M.replace("w","w#"),O="\\["+L+"*("+M+")(?:"+L+"*([*^$|!~]?=)"+L+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+N+"))|)"+L+"*\\]",P=":("+M+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+O+")*)|.*)\\)|)",Q=new RegExp(L+"+","g"),R=new RegExp("^"+L+"+|((?:^|[^\\\\])(?:\\\\.)*)"+L+"+$","g"),S=new RegExp("^"+L+"*,"+L+"*"),T=new RegExp("^"+L+"*([>+~]|"+L+")"+L+"*"),U=new RegExp("="+L+"*([^\\]'\"]*?)"+L+"*\\]","g"),V=new RegExp(P),W=new RegExp("^"+N+"$"),X={ID:new RegExp("^#("+M+")"),CLASS:new RegExp("^\\.("+M+")"),TAG:new RegExp("^("+M.replace("w","w*")+")"),ATTR:new RegExp("^"+O),PSEUDO:new RegExp("^"+P),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+L+"*(even|odd|(([+-]|)(\\d*)n|)"+L+"*(?:([+-]|)"+L+"*(\\d+)|))"+L+"*\\)|)","i"),bool:new RegExp("^(?:"+K+")$","i"),needsContext:new RegExp("^"+L+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+L+"*((?:-\\d)?\\d*)"+L+"*\\)|)(?=[^-]|$)","i")},Y=/^(?:input|select|textarea|button)$/i,Z=/^h\d$/i,$=/^[^{]+\{\s*\[native \w/,_=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ab=/[+~]/,bb=/'|\\/g,cb=new RegExp("\\\\([\\da-f]{1,6}"+L+"?|("+L+")|.)","ig"),db=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},eb=function(){m()};try{H.apply(E=I.call(v.childNodes),v.childNodes),E[v.childNodes.length].nodeType}catch(fb){H={apply:E.length?function(a,b){G.apply(a,I.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function gb(a,b,d,e){var f,h,j,k,l,o,r,s,w,x;if((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,d=d||[],k=b.nodeType,"string"!=typeof a||!a||1!==k&&9!==k&&11!==k)return d;if(!e&&p){if(11!==k&&(f=_.exec(a)))if(j=f[1]){if(9===k){if(h=b.getElementById(j),!h||!h.parentNode)return d;if(h.id===j)return d.push(h),d}else if(b.ownerDocument&&(h=b.ownerDocument.getElementById(j))&&t(b,h)&&h.id===j)return d.push(h),d}else{if(f[2])return H.apply(d,b.getElementsByTagName(a)),d;if((j=f[3])&&c.getElementsByClassName)return H.apply(d,b.getElementsByClassName(j)),d}if(c.qsa&&(!q||!q.test(a))){if(s=r=u,w=b,x=1!==k&&a,1===k&&"object"!==b.nodeName.toLowerCase()){o=g(a),(r=b.getAttribute("id"))?s=r.replace(bb,"\\$&"):b.setAttribute("id",s),s="[id='"+s+"'] ",l=o.length;while(l--)o[l]=s+rb(o[l]);w=ab.test(a)&&pb(b.parentNode)||b,x=o.join(",")}if(x)try{return H.apply(d,w.querySelectorAll(x)),d}catch(y){}finally{r||b.removeAttribute("id")}}}return i(a.replace(R,"$1"),b,d,e)}function hb(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ib(a){return a[u]=!0,a}function jb(a){var b=n.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function kb(a,b){var c=a.split("|"),e=a.length;while(e--)d.attrHandle[c[e]]=b}function lb(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||C)-(~a.sourceIndex||C);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function mb(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function nb(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function ob(a){return ib(function(b){return b=+b,ib(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function pb(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=gb.support={},f=gb.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},m=gb.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=g.documentElement,e=g.defaultView,e&&e!==e.top&&(e.addEventListener?e.addEventListener("unload",eb,!1):e.attachEvent&&e.attachEvent("onunload",eb)),p=!f(g),c.attributes=jb(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=jb(function(a){return a.appendChild(g.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=$.test(g.getElementsByClassName),c.getById=jb(function(a){return o.appendChild(a).id=u,!g.getElementsByName||!g.getElementsByName(u).length}),c.getById?(d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c&&c.parentNode?[c]:[]}},d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElementsByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return p?b.getElementsByClassName(a):void 0},r=[],q=[],(c.qsa=$.test(g.querySelectorAll))&&(jb(function(a){o.appendChild(a).innerHTML="",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+L+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+L+"*(?:value|"+K+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),jb(function(a){var b=g.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+L+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=$.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&jb(function(a){c.disconnectedMatch=s.call(a,"div"),s.call(a,"[s!='']:x"),r.push("!=",P)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=$.test(o.compareDocumentPosition),t=b||$.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===g||a.ownerDocument===v&&t(v,a)?-1:b===g||b.ownerDocument===v&&t(v,b)?1:k?J(k,a)-J(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,h=[a],i=[b];if(!e||!f)return a===g?-1:b===g?1:e?-1:f?1:k?J(k,a)-J(k,b):0;if(e===f)return lb(a,b);c=a;while(c=c.parentNode)h.unshift(c);c=b;while(c=c.parentNode)i.unshift(c);while(h[d]===i[d])d++;return d?lb(h[d],i[d]):h[d]===v?-1:i[d]===v?1:0},g):n},gb.matches=function(a,b){return gb(a,null,null,b)},gb.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(U,"='$1']"),!(!c.matchesSelector||!p||r&&r.test(b)||q&&q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return gb(b,n,null,[a]).length>0},gb.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},gb.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&D.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},gb.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},gb.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=gb.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=gb.selectors={cacheLength:50,createPseudo:ib,match:X,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(cb,db),a[3]=(a[3]||a[4]||a[5]||"").replace(cb,db),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||gb.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&gb.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return X.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&V.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(cb,db).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+L+")"+a+"("+L+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=gb.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(Q," ")+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h;if(q){if(f){while(p){l=b;while(l=l[p])if(h?l.nodeName.toLowerCase()===r:1===l.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){k=q[u]||(q[u]={}),j=k[a]||[],n=j[0]===w&&j[1],m=j[0]===w&&j[2],l=n&&q.childNodes[n];while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if(1===l.nodeType&&++m&&l===b){k[a]=[w,n,m];break}}else if(s&&(j=(b[u]||(b[u]={}))[a])&&j[0]===w)m=j[1];else while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if((h?l.nodeName.toLowerCase()===r:1===l.nodeType)&&++m&&(s&&((l[u]||(l[u]={}))[a]=[w,m]),l===b))break;return m-=e,m===d||m%d===0&&m/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||gb.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ib(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=J(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ib(function(a){var b=[],c=[],d=h(a.replace(R,"$1"));return d[u]?ib(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ib(function(a){return function(b){return gb(a,b).length>0}}),contains:ib(function(a){return a=a.replace(cb,db),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ib(function(a){return W.test(a||"")||gb.error("unsupported lang: "+a),a=a.replace(cb,db).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return Z.test(a.nodeName)},input:function(a){return Y.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:ob(function(){return[0]}),last:ob(function(a,b){return[b-1]}),eq:ob(function(a,b,c){return[0>c?c+b:c]}),even:ob(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:ob(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:ob(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:ob(function(a,b,c){for(var d=0>c?c+b:c;++db;b++)d+=a[b].value;return d}function sb(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=x++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j=[w,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(i=b[u]||(b[u]={}),(h=i[d])&&h[0]===w&&h[1]===f)return j[2]=h[2];if(i[d]=j,j[2]=a(b,c,g))return!0}}}function tb(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function ub(a,b,c){for(var d=0,e=b.length;e>d;d++)gb(a,b[d],c);return c}function vb(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(!c||c(f,d,e))&&(g.push(f),j&&b.push(h));return g}function wb(a,b,c,d,e,f){return d&&!d[u]&&(d=wb(d)),e&&!e[u]&&(e=wb(e,f)),ib(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||ub(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:vb(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=vb(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?J(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=vb(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):H.apply(g,r)})}function xb(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=sb(function(a){return a===b},h,!0),l=sb(function(a){return J(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];f>i;i++)if(c=d.relative[a[i].type])m=[sb(tb(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;f>e;e++)if(d.relative[a[e].type])break;return wb(i>1&&tb(m),i>1&&rb(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(R,"$1"),c,e>i&&xb(a.slice(i,e)),f>e&&xb(a=a.slice(e)),f>e&&rb(a))}m.push(c)}return tb(m)}function yb(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,m,o,p=0,q="0",r=f&&[],s=[],t=j,u=f||e&&d.find.TAG("*",k),v=w+=null==t?1:Math.random()||.1,x=u.length;for(k&&(j=g!==n&&g);q!==x&&null!=(l=u[q]);q++){if(e&&l){m=0;while(o=a[m++])if(o(l,g,h)){i.push(l);break}k&&(w=v)}c&&((l=!o&&l)&&p--,f&&r.push(l))}if(p+=q,c&&q!==p){m=0;while(o=b[m++])o(r,s,g,h);if(f){if(p>0)while(q--)r[q]||s[q]||(s[q]=F.call(i));s=vb(s)}H.apply(i,s),k&&!f&&s.length>0&&p+b.length>1&&gb.uniqueSort(i)}return k&&(w=v,j=t),r};return c?ib(f):f}return h=gb.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=xb(b[c]),f[u]?d.push(f):e.push(f);f=A(a,yb(e,d)),f.selector=a}return f},i=gb.select=function(a,b,e,f){var i,j,k,l,m,n="function"==typeof a&&a,o=!f&&g(a=n.selector||a);if(e=e||[],1===o.length){if(j=o[0]=o[0].slice(0),j.length>2&&"ID"===(k=j[0]).type&&c.getById&&9===b.nodeType&&p&&d.relative[j[1].type]){if(b=(d.find.ID(k.matches[0].replace(cb,db),b)||[])[0],!b)return e;n&&(b=b.parentNode),a=a.slice(j.shift().value.length)}i=X.needsContext.test(a)?0:j.length;while(i--){if(k=j[i],d.relative[l=k.type])break;if((m=d.find[l])&&(f=m(k.matches[0].replace(cb,db),ab.test(j[0].type)&&pb(b.parentNode)||b))){if(j.splice(i,1),a=f.length&&rb(j),!a)return H.apply(e,f),e;break}}}return(n||h(a,o))(f,b,!p,e,ab.test(a)&&pb(b.parentNode)||b),e},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=jb(function(a){return 1&a.compareDocumentPosition(n.createElement("div"))}),jb(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||kb("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&jb(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||kb("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),jb(function(a){return null==a.getAttribute("disabled")})||kb(K,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),gb}(a);n.find=t,n.expr=t.selectors,n.expr[":"]=n.expr.pseudos,n.unique=t.uniqueSort,n.text=t.getText,n.isXMLDoc=t.isXML,n.contains=t.contains;var u=n.expr.match.needsContext,v=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,w=/^.[^:#\[\.,]*$/;function x(a,b,c){if(n.isFunction(b))return n.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return n.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(w.test(b))return n.filter(b,a,c);b=n.filter(b,a)}return n.grep(a,function(a){return g.call(b,a)>=0!==c})}n.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?n.find.matchesSelector(d,a)?[d]:[]:n.find.matches(a,n.grep(b,function(a){return 1===a.nodeType}))},n.fn.extend({find:function(a){var b,c=this.length,d=[],e=this;if("string"!=typeof a)return this.pushStack(n(a).filter(function(){for(b=0;c>b;b++)if(n.contains(e[b],this))return!0}));for(b=0;c>b;b++)n.find(a,e[b],d);return d=this.pushStack(c>1?n.unique(d):d),d.selector=this.selector?this.selector+" "+a:a,d},filter:function(a){return this.pushStack(x(this,a||[],!1))},not:function(a){return this.pushStack(x(this,a||[],!0))},is:function(a){return!!x(this,"string"==typeof a&&u.test(a)?n(a):a||[],!1).length}});var y,z=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,A=n.fn.init=function(a,b){var c,d;if(!a)return this;if("string"==typeof a){if(c="<"===a[0]&&">"===a[a.length-1]&&a.length>=3?[null,a,null]:z.exec(a),!c||!c[1]&&b)return!b||b.jquery?(b||y).find(a):this.constructor(b).find(a);if(c[1]){if(b=b instanceof n?b[0]:b,n.merge(this,n.parseHTML(c[1],b&&b.nodeType?b.ownerDocument||b:l,!0)),v.test(c[1])&&n.isPlainObject(b))for(c in b)n.isFunction(this[c])?this[c](b[c]):this.attr(c,b[c]);return this}return d=l.getElementById(c[2]),d&&d.parentNode&&(this.length=1,this[0]=d),this.context=l,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):n.isFunction(a)?"undefined"!=typeof y.ready?y.ready(a):a(n):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),n.makeArray(a,this))};A.prototype=n.fn,y=n(l);var B=/^(?:parents|prev(?:Until|All))/,C={children:!0,contents:!0,next:!0,prev:!0};n.extend({dir:function(a,b,c){var d=[],e=void 0!==c;while((a=a[b])&&9!==a.nodeType)if(1===a.nodeType){if(e&&n(a).is(c))break;d.push(a)}return d},sibling:function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c}}),n.fn.extend({has:function(a){var b=n(a,this),c=b.length;return this.filter(function(){for(var a=0;c>a;a++)if(n.contains(this,b[a]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=u.test(a)||"string"!=typeof a?n(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&n.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?n.unique(f):f)},index:function(a){return a?"string"==typeof a?g.call(n(a),this[0]):g.call(this,a.jquery?a[0]:a):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(n.unique(n.merge(this.get(),n(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function D(a,b){while((a=a[b])&&1!==a.nodeType);return a}n.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return n.dir(a,"parentNode")},parentsUntil:function(a,b,c){return n.dir(a,"parentNode",c)},next:function(a){return D(a,"nextSibling")},prev:function(a){return D(a,"previousSibling")},nextAll:function(a){return n.dir(a,"nextSibling")},prevAll:function(a){return n.dir(a,"previousSibling")},nextUntil:function(a,b,c){return n.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return n.dir(a,"previousSibling",c)},siblings:function(a){return n.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return n.sibling(a.firstChild)},contents:function(a){return a.contentDocument||n.merge([],a.childNodes)}},function(a,b){n.fn[a]=function(c,d){var e=n.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=n.filter(d,e)),this.length>1&&(C[a]||n.unique(e),B.test(a)&&e.reverse()),this.pushStack(e)}});var E=/\S+/g,F={};function G(a){var b=F[a]={};return n.each(a.match(E)||[],function(a,c){b[c]=!0}),b}n.Callbacks=function(a){a="string"==typeof a?F[a]||G(a):n.extend({},a);var b,c,d,e,f,g,h=[],i=!a.once&&[],j=function(l){for(b=a.memory&&l,c=!0,g=e||0,e=0,f=h.length,d=!0;h&&f>g;g++)if(h[g].apply(l[0],l[1])===!1&&a.stopOnFalse){b=!1;break}d=!1,h&&(i?i.length&&j(i.shift()):b?h=[]:k.disable())},k={add:function(){if(h){var c=h.length;!function g(b){n.each(b,function(b,c){var d=n.type(c);"function"===d?a.unique&&k.has(c)||h.push(c):c&&c.length&&"string"!==d&&g(c)})}(arguments),d?f=h.length:b&&(e=c,j(b))}return this},remove:function(){return h&&n.each(arguments,function(a,b){var c;while((c=n.inArray(b,h,c))>-1)h.splice(c,1),d&&(f>=c&&f--,g>=c&&g--)}),this},has:function(a){return a?n.inArray(a,h)>-1:!(!h||!h.length)},empty:function(){return h=[],f=0,this},disable:function(){return h=i=b=void 0,this},disabled:function(){return!h},lock:function(){return i=void 0,b||k.disable(),this},locked:function(){return!i},fireWith:function(a,b){return!h||c&&!i||(b=b||[],b=[a,b.slice?b.slice():b],d?i.push(b):j(b)),this},fire:function(){return k.fireWith(this,arguments),this},fired:function(){return!!c}};return k},n.extend({Deferred:function(a){var b=[["resolve","done",n.Callbacks("once memory"),"resolved"],["reject","fail",n.Callbacks("once memory"),"rejected"],["notify","progress",n.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return n.Deferred(function(c){n.each(b,function(b,f){var g=n.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&n.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?n.extend(a,d):d}},e={};return d.pipe=d.then,n.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=d.call(arguments),e=c.length,f=1!==e||a&&n.isFunction(a.promise)?e:0,g=1===f?a:n.Deferred(),h=function(a,b,c){return function(e){b[a]=this,c[a]=arguments.length>1?d.call(arguments):e,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(e>1)for(i=new Array(e),j=new Array(e),k=new Array(e);e>b;b++)c[b]&&n.isFunction(c[b].promise)?c[b].promise().done(h(b,k,c)).fail(g.reject).progress(h(b,j,i)):--f;return f||g.resolveWith(k,c),g.promise()}});var H;n.fn.ready=function(a){return n.ready.promise().done(a),this},n.extend({isReady:!1,readyWait:1,holdReady:function(a){a?n.readyWait++:n.ready(!0)},ready:function(a){(a===!0?--n.readyWait:n.isReady)||(n.isReady=!0,a!==!0&&--n.readyWait>0||(H.resolveWith(l,[n]),n.fn.triggerHandler&&(n(l).triggerHandler("ready"),n(l).off("ready"))))}});function I(){l.removeEventListener("DOMContentLoaded",I,!1),a.removeEventListener("load",I,!1),n.ready()}n.ready.promise=function(b){return H||(H=n.Deferred(),"complete"===l.readyState?setTimeout(n.ready):(l.addEventListener("DOMContentLoaded",I,!1),a.addEventListener("load",I,!1))),H.promise(b)},n.ready.promise();var J=n.access=function(a,b,c,d,e,f,g){var h=0,i=a.length,j=null==c;if("object"===n.type(c)){e=!0;for(h in c)n.access(a,b,h,c[h],!0,f,g)}else if(void 0!==d&&(e=!0,n.isFunction(d)||(g=!0),j&&(g?(b.call(a,d),b=null):(j=b,b=function(a,b,c){return j.call(n(a),c)})),b))for(;i>h;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f};n.acceptData=function(a){return 1===a.nodeType||9===a.nodeType||!+a.nodeType};function K(){Object.defineProperty(this.cache={},0,{get:function(){return{}}}),this.expando=n.expando+K.uid++}K.uid=1,K.accepts=n.acceptData,K.prototype={key:function(a){if(!K.accepts(a))return 0;var b={},c=a[this.expando];if(!c){c=K.uid++;try{b[this.expando]={value:c},Object.defineProperties(a,b)}catch(d){b[this.expando]=c,n.extend(a,b)}}return this.cache[c]||(this.cache[c]={}),c},set:function(a,b,c){var d,e=this.key(a),f=this.cache[e];if("string"==typeof b)f[b]=c;else if(n.isEmptyObject(f))n.extend(this.cache[e],b);else for(d in b)f[d]=b[d];return f},get:function(a,b){var c=this.cache[this.key(a)];return void 0===b?c:c[b]},access:function(a,b,c){var d;return void 0===b||b&&"string"==typeof b&&void 0===c?(d=this.get(a,b),void 0!==d?d:this.get(a,n.camelCase(b))):(this.set(a,b,c),void 0!==c?c:b)},remove:function(a,b){var c,d,e,f=this.key(a),g=this.cache[f];if(void 0===b)this.cache[f]={};else{n.isArray(b)?d=b.concat(b.map(n.camelCase)):(e=n.camelCase(b),b in g?d=[b,e]:(d=e,d=d in g?[d]:d.match(E)||[])),c=d.length;while(c--)delete g[d[c]]}},hasData:function(a){return!n.isEmptyObject(this.cache[a[this.expando]]||{})},discard:function(a){a[this.expando]&&delete this.cache[a[this.expando]]}};var L=new K,M=new K,N=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,O=/([A-Z])/g;function P(a,b,c){var d;if(void 0===c&&1===a.nodeType)if(d="data-"+b.replace(O,"-$1").toLowerCase(),c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:N.test(c)?n.parseJSON(c):c}catch(e){}M.set(a,b,c)}else c=void 0;return c}n.extend({hasData:function(a){return M.hasData(a)||L.hasData(a)},data:function(a,b,c){return M.access(a,b,c) -},removeData:function(a,b){M.remove(a,b)},_data:function(a,b,c){return L.access(a,b,c)},_removeData:function(a,b){L.remove(a,b)}}),n.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=M.get(f),1===f.nodeType&&!L.get(f,"hasDataAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=n.camelCase(d.slice(5)),P(f,d,e[d])));L.set(f,"hasDataAttrs",!0)}return e}return"object"==typeof a?this.each(function(){M.set(this,a)}):J(this,function(b){var c,d=n.camelCase(a);if(f&&void 0===b){if(c=M.get(f,a),void 0!==c)return c;if(c=M.get(f,d),void 0!==c)return c;if(c=P(f,d,void 0),void 0!==c)return c}else this.each(function(){var c=M.get(this,d);M.set(this,d,b),-1!==a.indexOf("-")&&void 0!==c&&M.set(this,a,b)})},null,b,arguments.length>1,null,!0)},removeData:function(a){return this.each(function(){M.remove(this,a)})}}),n.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=L.get(a,b),c&&(!d||n.isArray(c)?d=L.access(a,b,n.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=n.queue(a,b),d=c.length,e=c.shift(),f=n._queueHooks(a,b),g=function(){n.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return L.get(a,c)||L.access(a,c,{empty:n.Callbacks("once memory").add(function(){L.remove(a,[b+"queue",c])})})}}),n.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthx",k.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue}();var U="undefined";k.focusinBubbles="onfocusin"in a;var V=/^key/,W=/^(?:mouse|pointer|contextmenu)|click/,X=/^(?:focusinfocus|focusoutblur)$/,Y=/^([^.]*)(?:\.(.+)|)$/;function Z(){return!0}function $(){return!1}function _(){try{return l.activeElement}catch(a){}}n.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=L.get(a);if(r){c.handler&&(f=c,c=f.handler,e=f.selector),c.guid||(c.guid=n.guid++),(i=r.events)||(i=r.events={}),(g=r.handle)||(g=r.handle=function(b){return typeof n!==U&&n.event.triggered!==b.type?n.event.dispatch.apply(a,arguments):void 0}),b=(b||"").match(E)||[""],j=b.length;while(j--)h=Y.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o&&(l=n.event.special[o]||{},o=(e?l.delegateType:l.bindType)||o,l=n.event.special[o]||{},k=n.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&n.expr.match.needsContext.test(e),namespace:p.join(".")},f),(m=i[o])||(m=i[o]=[],m.delegateCount=0,l.setup&&l.setup.call(a,d,p,g)!==!1||a.addEventListener&&a.addEventListener(o,g,!1)),l.add&&(l.add.call(a,k),k.handler.guid||(k.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,k):m.push(k),n.event.global[o]=!0)}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=L.hasData(a)&&L.get(a);if(r&&(i=r.events)){b=(b||"").match(E)||[""],j=b.length;while(j--)if(h=Y.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=n.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,m=i[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),g=f=m.length;while(f--)k=m[f],!e&&q!==k.origType||c&&c.guid!==k.guid||h&&!h.test(k.namespace)||d&&d!==k.selector&&("**"!==d||!k.selector)||(m.splice(f,1),k.selector&&m.delegateCount--,l.remove&&l.remove.call(a,k));g&&!m.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||n.removeEvent(a,o,r.handle),delete i[o])}else for(o in i)n.event.remove(a,o+b[j],c,d,!0);n.isEmptyObject(i)&&(delete r.handle,L.remove(a,"events"))}},trigger:function(b,c,d,e){var f,g,h,i,k,m,o,p=[d||l],q=j.call(b,"type")?b.type:b,r=j.call(b,"namespace")?b.namespace.split("."):[];if(g=h=d=d||l,3!==d.nodeType&&8!==d.nodeType&&!X.test(q+n.event.triggered)&&(q.indexOf(".")>=0&&(r=q.split("."),q=r.shift(),r.sort()),k=q.indexOf(":")<0&&"on"+q,b=b[n.expando]?b:new n.Event(q,"object"==typeof b&&b),b.isTrigger=e?2:3,b.namespace=r.join("."),b.namespace_re=b.namespace?new RegExp("(^|\\.)"+r.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=d),c=null==c?[b]:n.makeArray(c,[b]),o=n.event.special[q]||{},e||!o.trigger||o.trigger.apply(d,c)!==!1)){if(!e&&!o.noBubble&&!n.isWindow(d)){for(i=o.delegateType||q,X.test(i+q)||(g=g.parentNode);g;g=g.parentNode)p.push(g),h=g;h===(d.ownerDocument||l)&&p.push(h.defaultView||h.parentWindow||a)}f=0;while((g=p[f++])&&!b.isPropagationStopped())b.type=f>1?i:o.bindType||q,m=(L.get(g,"events")||{})[b.type]&&L.get(g,"handle"),m&&m.apply(g,c),m=k&&g[k],m&&m.apply&&n.acceptData(g)&&(b.result=m.apply(g,c),b.result===!1&&b.preventDefault());return b.type=q,e||b.isDefaultPrevented()||o._default&&o._default.apply(p.pop(),c)!==!1||!n.acceptData(d)||k&&n.isFunction(d[q])&&!n.isWindow(d)&&(h=d[k],h&&(d[k]=null),n.event.triggered=q,d[q](),n.event.triggered=void 0,h&&(d[k]=h)),b.result}},dispatch:function(a){a=n.event.fix(a);var b,c,e,f,g,h=[],i=d.call(arguments),j=(L.get(this,"events")||{})[a.type]||[],k=n.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=n.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,c=0;while((g=f.handlers[c++])&&!a.isImmediatePropagationStopped())(!a.namespace_re||a.namespace_re.test(g.namespace))&&(a.handleObj=g,a.data=g.data,e=((n.event.special[g.origType]||{}).handle||g.handler).apply(f.elem,i),void 0!==e&&(a.result=e)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&(!a.button||"click"!==a.type))for(;i!==this;i=i.parentNode||this)if(i.disabled!==!0||"click"!==a.type){for(d=[],c=0;h>c;c++)f=b[c],e=f.selector+" ",void 0===d[e]&&(d[e]=f.needsContext?n(e,this).index(i)>=0:n.find(e,this,null,[i]).length),d[e]&&d.push(f);d.length&&g.push({elem:i,handlers:d})}return h]*)\/>/gi,bb=/<([\w:]+)/,cb=/<|&#?\w+;/,db=/<(?:script|style|link)/i,eb=/checked\s*(?:[^=]|=\s*.checked.)/i,fb=/^$|\/(?:java|ecma)script/i,gb=/^true\/(.*)/,hb=/^\s*\s*$/g,ib={option:[1,""],thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};ib.optgroup=ib.option,ib.tbody=ib.tfoot=ib.colgroup=ib.caption=ib.thead,ib.th=ib.td;function jb(a,b){return n.nodeName(a,"table")&&n.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function kb(a){return a.type=(null!==a.getAttribute("type"))+"/"+a.type,a}function lb(a){var b=gb.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function mb(a,b){for(var c=0,d=a.length;d>c;c++)L.set(a[c],"globalEval",!b||L.get(b[c],"globalEval"))}function nb(a,b){var c,d,e,f,g,h,i,j;if(1===b.nodeType){if(L.hasData(a)&&(f=L.access(a),g=L.set(b,f),j=f.events)){delete g.handle,g.events={};for(e in j)for(c=0,d=j[e].length;d>c;c++)n.event.add(b,e,j[e][c])}M.hasData(a)&&(h=M.access(a),i=n.extend({},h),M.set(b,i))}}function ob(a,b){var c=a.getElementsByTagName?a.getElementsByTagName(b||"*"):a.querySelectorAll?a.querySelectorAll(b||"*"):[];return void 0===b||b&&n.nodeName(a,b)?n.merge([a],c):c}function pb(a,b){var c=b.nodeName.toLowerCase();"input"===c&&T.test(a.type)?b.checked=a.checked:("input"===c||"textarea"===c)&&(b.defaultValue=a.defaultValue)}n.extend({clone:function(a,b,c){var d,e,f,g,h=a.cloneNode(!0),i=n.contains(a.ownerDocument,a);if(!(k.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||n.isXMLDoc(a)))for(g=ob(h),f=ob(a),d=0,e=f.length;e>d;d++)pb(f[d],g[d]);if(b)if(c)for(f=f||ob(a),g=g||ob(h),d=0,e=f.length;e>d;d++)nb(f[d],g[d]);else nb(a,h);return g=ob(h,"script"),g.length>0&&mb(g,!i&&ob(a,"script")),h},buildFragment:function(a,b,c,d){for(var e,f,g,h,i,j,k=b.createDocumentFragment(),l=[],m=0,o=a.length;o>m;m++)if(e=a[m],e||0===e)if("object"===n.type(e))n.merge(l,e.nodeType?[e]:e);else if(cb.test(e)){f=f||k.appendChild(b.createElement("div")),g=(bb.exec(e)||["",""])[1].toLowerCase(),h=ib[g]||ib._default,f.innerHTML=h[1]+e.replace(ab,"<$1>")+h[2],j=h[0];while(j--)f=f.lastChild;n.merge(l,f.childNodes),f=k.firstChild,f.textContent=""}else l.push(b.createTextNode(e));k.textContent="",m=0;while(e=l[m++])if((!d||-1===n.inArray(e,d))&&(i=n.contains(e.ownerDocument,e),f=ob(k.appendChild(e),"script"),i&&mb(f),c)){j=0;while(e=f[j++])fb.test(e.type||"")&&c.push(e)}return k},cleanData:function(a){for(var b,c,d,e,f=n.event.special,g=0;void 0!==(c=a[g]);g++){if(n.acceptData(c)&&(e=c[L.expando],e&&(b=L.cache[e]))){if(b.events)for(d in b.events)f[d]?n.event.remove(c,d):n.removeEvent(c,d,b.handle);L.cache[e]&&delete L.cache[e]}delete M.cache[c[M.expando]]}}}),n.fn.extend({text:function(a){return J(this,function(a){return void 0===a?n.text(this):this.empty().each(function(){(1===this.nodeType||11===this.nodeType||9===this.nodeType)&&(this.textContent=a)})},null,a,arguments.length)},append:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=jb(this,a);b.appendChild(a)}})},prepend:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=jb(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},remove:function(a,b){for(var c,d=a?n.filter(a,this):this,e=0;null!=(c=d[e]);e++)b||1!==c.nodeType||n.cleanData(ob(c)),c.parentNode&&(b&&n.contains(c.ownerDocument,c)&&mb(ob(c,"script")),c.parentNode.removeChild(c));return this},empty:function(){for(var a,b=0;null!=(a=this[b]);b++)1===a.nodeType&&(n.cleanData(ob(a,!1)),a.textContent="");return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return n.clone(this,a,b)})},html:function(a){return J(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a&&1===b.nodeType)return b.innerHTML;if("string"==typeof a&&!db.test(a)&&!ib[(bb.exec(a)||["",""])[1].toLowerCase()]){a=a.replace(ab,"<$1>");try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(n.cleanData(ob(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=arguments[0];return this.domManip(arguments,function(b){a=this.parentNode,n.cleanData(ob(this)),a&&a.replaceChild(b,this)}),a&&(a.length||a.nodeType)?this:this.remove()},detach:function(a){return this.remove(a,!0)},domManip:function(a,b){a=e.apply([],a);var c,d,f,g,h,i,j=0,l=this.length,m=this,o=l-1,p=a[0],q=n.isFunction(p);if(q||l>1&&"string"==typeof p&&!k.checkClone&&eb.test(p))return this.each(function(c){var d=m.eq(c);q&&(a[0]=p.call(this,c,d.html())),d.domManip(a,b)});if(l&&(c=n.buildFragment(a,this[0].ownerDocument,!1,this),d=c.firstChild,1===c.childNodes.length&&(c=d),d)){for(f=n.map(ob(c,"script"),kb),g=f.length;l>j;j++)h=c,j!==o&&(h=n.clone(h,!0,!0),g&&n.merge(f,ob(h,"script"))),b.call(this[j],h,j);if(g)for(i=f[f.length-1].ownerDocument,n.map(f,lb),j=0;g>j;j++)h=f[j],fb.test(h.type||"")&&!L.access(h,"globalEval")&&n.contains(i,h)&&(h.src?n._evalUrl&&n._evalUrl(h.src):n.globalEval(h.textContent.replace(hb,"")))}return this}}),n.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){n.fn[a]=function(a){for(var c,d=[],e=n(a),g=e.length-1,h=0;g>=h;h++)c=h===g?this:this.clone(!0),n(e[h])[b](c),f.apply(d,c.get());return this.pushStack(d)}});var qb,rb={};function sb(b,c){var d,e=n(c.createElement(b)).appendTo(c.body),f=a.getDefaultComputedStyle&&(d=a.getDefaultComputedStyle(e[0]))?d.display:n.css(e[0],"display");return e.detach(),f}function tb(a){var b=l,c=rb[a];return c||(c=sb(a,b),"none"!==c&&c||(qb=(qb||n("