From 6c28ac260c2ce4bf93737e53ea3297bff08924c7 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 7 Jan 2016 04:26:29 +0000 Subject: copyrights --- synapse/storage/appservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/storage/appservice.py') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 39b7881c40..eab58d9ce9 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd +# Copyright 2015, 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -- cgit 1.4.1 From 244b356a37ada45d2a4e6aec1f08986aaa7eaaa1 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 13 Jan 2016 17:03:58 +0000 Subject: Delete unused code --- synapse/storage/appservice.py | 59 ------------------------------------------- 1 file changed, 59 deletions(-) (limited to 'synapse/storage/appservice.py') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index eab58d9ce9..1a2b4678a2 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -15,7 +15,6 @@ import logging import urllib import yaml -from simplejson import JSONDecodeError import simplejson as json from twisted.internet import defer @@ -144,64 +143,6 @@ class ApplicationServiceStore(SQLBaseStore): return rooms_for_user_matching_user_id - def _parse_services_dict(self, results): - # SQL results in the form: - # [ - # { - # 'regex': "something", - # 'url': "something", - # 'namespace': enum, - # 'as_id': 0, - # 'token': "something", - # 'hs_token': "otherthing", - # 'id': 0 - # } - # ] - services = {} - for res in results: - as_token = res["token"] - if as_token is None: - continue - if as_token not in services: - # add the service - services[as_token] = { - "id": res["id"], - "url": res["url"], - "token": as_token, - "hs_token": res["hs_token"], - "sender": res["sender"], - "namespaces": { - ApplicationService.NS_USERS: [], - ApplicationService.NS_ALIASES: [], - ApplicationService.NS_ROOMS: [] - } - } - # add the namespace regex if one exists - ns_int = res["namespace"] - if ns_int is None: - continue - try: - services[as_token]["namespaces"][ - ApplicationService.NS_LIST[ns_int]].append( - json.loads(res["regex"]) - ) - except IndexError: - logger.error("Bad namespace enum '%s'. %s", ns_int, res) - except JSONDecodeError: - logger.error("Bad regex object '%s'", res["regex"]) - - service_list = [] - for service in services.values(): - service_list.append(ApplicationService( - token=service["token"], - url=service["url"], - namespaces=service["namespaces"], - hs_token=service["hs_token"], - sender=service["sender"], - id=service["id"] - )) - return service_list - def _load_appservice(self, as_info): required_string_fields = [ "url", "as_token", "hs_token", "sender_localpart" -- cgit 1.4.1 From f6fcff360250eb362986835c195a96825567a03d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 13 Jan 2016 17:09:24 +0000 Subject: Don't start server if ASes are invalidly configured --- synapse/storage/appservice.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse/storage/appservice.py') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index eab58d9ce9..25a6f14f40 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -265,6 +265,7 @@ class ApplicationServiceStore(SQLBaseStore): except Exception as e: logger.error("Failed to load appservice from '%s'", config_file) logger.exception(e) + raise class ApplicationServiceTransactionStore(SQLBaseStore): -- cgit 1.4.1 From 2680043bc6a64053b93b9bab144aeb5f45007976 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 14 Jan 2016 14:34:01 +0000 Subject: Require ID and as_token be unique for ASs Defaults ID to as_token if not specified. This will change when IDs are fully supported. --- synapse/storage/appservice.py | 26 +++++++++- tests/appservice/test_appservice.py | 1 + tests/storage/test_appservice.py | 101 ++++++++++++++++++++++++++++++------ 3 files changed, 111 insertions(+), 17 deletions(-) (limited to 'synapse/storage/appservice.py') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index f4bc457eca..b5aa55c0a3 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -20,6 +20,7 @@ from twisted.internet import defer from synapse.api.constants import Membership from synapse.appservice import ApplicationService, AppServiceTransaction +from synapse.config._base import ConfigError from synapse.storage.roommember import RoomsForUser from synapse.types import UserID from ._base import SQLBaseStore @@ -145,6 +146,7 @@ class ApplicationServiceStore(SQLBaseStore): def _load_appservice(self, as_info): required_string_fields = [ + # TODO: Add id here when it's stable to release "url", "as_token", "hs_token", "sender_localpart" ] for field in required_string_fields: @@ -186,7 +188,7 @@ class ApplicationServiceStore(SQLBaseStore): namespaces=as_info["namespaces"], hs_token=as_info["hs_token"], sender=user_id, - id=as_info["as_token"] # the token is the only unique thing here + id=as_info["id"] if "id" in as_info else as_info["as_token"], ) def _populate_appservice_cache(self, config_files): @@ -197,10 +199,32 @@ class ApplicationServiceStore(SQLBaseStore): ) return + # Dicts of value -> filename + seen_as_tokens = {} + seen_ids = {} + for config_file in config_files: try: with open(config_file, 'r') as f: appservice = self._load_appservice(yaml.load(f)) + if appservice.id in seen_ids: + raise ConfigError( + "Cannot reuse ID across application services: " + "%s (files: %s, %s)" % ( + appservice.id, config_file, seen_ids[appservice.id], + ) + ) + seen_ids[appservice.id] = config_file + if appservice.token in seen_as_tokens: + raise ConfigError( + "Cannot reuse as_token across application services: " + "%s (files: %s, %s)" % ( + appservice.token, + config_file, + seen_as_tokens[appservice.token], + ) + ) + seen_as_tokens[appservice.token] = config_file logger.info("Loaded application service: %s", appservice) self.services_cache.append(appservice) except Exception as e: diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 191c420c4d..ef48bbc296 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -29,6 +29,7 @@ class ApplicationServiceTestCase(unittest.TestCase): def setUp(self): self.service = ApplicationService( + id="unique_identifier", url="some_url", token="some_token", namespaces={ diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index a5a464640f..5abecdf6e0 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -12,12 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import tempfile +from synapse.config._base import ConfigError from tests import unittest from twisted.internet import defer from tests.utils import setup_test_homeserver from synapse.appservice import ApplicationService, ApplicationServiceState -from synapse.server import HomeServer from synapse.storage.appservice import ( ApplicationServiceStore, ApplicationServiceTransactionStore ) @@ -26,7 +27,6 @@ import json import os import yaml from mock import Mock -from tests.utils import SQLiteMemoryDbPool, MockClock class ApplicationServiceStoreTestCase(unittest.TestCase): @@ -41,9 +41,16 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): self.as_token = "token1" self.as_url = "some_url" - self._add_appservice(self.as_token, self.as_url, "some_hs_token", "bob") - self._add_appservice("token2", "some_url", "some_hs_token", "bob") - self._add_appservice("token3", "some_url", "some_hs_token", "bob") + self.as_id = "as1" + self._add_appservice( + self.as_token, + self.as_id, + self.as_url, + "some_hs_token", + "bob" + ) + self._add_appservice("token2", "as2", "some_url", "some_hs_token", "bob") + self._add_appservice("token3", "as3", "some_url", "some_hs_token", "bob") # must be done after inserts self.store = ApplicationServiceStore(hs) @@ -55,9 +62,9 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): except: pass - def _add_appservice(self, as_token, url, hs_token, sender): + def _add_appservice(self, as_token, id, url, hs_token, sender): as_yaml = dict(url=url, as_token=as_token, hs_token=hs_token, - sender_localpart=sender, namespaces={}) + id=id, sender_localpart=sender, namespaces={}) # use the token as the filename with open(as_token, 'w') as outfile: outfile.write(yaml.dump(as_yaml)) @@ -74,6 +81,7 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): self.as_token ) self.assertEquals(stored_service.token, self.as_token) + self.assertEquals(stored_service.id, self.as_id) self.assertEquals(stored_service.url, self.as_url) self.assertEquals( stored_service.namespaces[ApplicationService.NS_ALIASES], @@ -110,34 +118,34 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): { "token": "token1", "url": "https://matrix-as.org", - "id": "token1" + "id": "id_1" }, { "token": "alpha_tok", "url": "https://alpha.com", - "id": "alpha_tok" + "id": "id_alpha" }, { "token": "beta_tok", "url": "https://beta.com", - "id": "beta_tok" + "id": "id_beta" }, { - "token": "delta_tok", - "url": "https://delta.com", - "id": "delta_tok" + "token": "gamma_tok", + "url": "https://gamma.com", + "id": "id_gamma" }, ] for s in self.as_list: - yield self._add_service(s["url"], s["token"]) + yield self._add_service(s["url"], s["token"], s["id"]) self.as_yaml_files = [] self.store = TestTransactionStore(hs) - def _add_service(self, url, as_token): + def _add_service(self, url, as_token, id): as_yaml = dict(url=url, as_token=as_token, hs_token="something", - sender_localpart="a_sender", namespaces={}) + id=id, sender_localpart="a_sender", namespaces={}) # use the token as the filename with open(as_token, 'w') as outfile: outfile.write(yaml.dump(as_yaml)) @@ -405,3 +413,64 @@ class TestTransactionStore(ApplicationServiceTransactionStore, def __init__(self, hs): super(TestTransactionStore, self).__init__(hs) + + +class ApplicationServiceStoreConfigTestCase(unittest.TestCase): + + def _write_config(self, suffix, **kwargs): + vals = { + "id": "id" + suffix, + "url": "url" + suffix, + "as_token": "as_token" + suffix, + "hs_token": "hs_token" + suffix, + "sender_localpart": "sender_localpart" + suffix, + "namespaces": {}, + } + vals.update(kwargs) + + _, path = tempfile.mkstemp(prefix="as_config") + with open(path, "w") as f: + f.write(yaml.dump(vals)) + return path + + @defer.inlineCallbacks + def test_unique_works(self): + f1 = self._write_config(suffix="1") + f2 = self._write_config(suffix="2") + + config = Mock(app_service_config_files=[f1, f2]) + hs = yield setup_test_homeserver(config=config) + + ApplicationServiceStore(hs) + + @defer.inlineCallbacks + def test_duplicate_ids(self): + f1 = self._write_config(id="id", suffix="1") + f2 = self._write_config(id="id", suffix="2") + + config = Mock(app_service_config_files=[f1, f2]) + hs = yield setup_test_homeserver(config=config) + + with self.assertRaises(ConfigError) as cm: + ApplicationServiceStore(hs) + + e = cm.exception + self.assertIn(f1, e.message) + self.assertIn(f2, e.message) + self.assertIn("id", e.message) + + @defer.inlineCallbacks + def test_duplicate_as_tokens(self): + f1 = self._write_config(as_token="as_token", suffix="1") + f2 = self._write_config(as_token="as_token", suffix="2") + + config = Mock(app_service_config_files=[f1, f2]) + hs = yield setup_test_homeserver(config=config) + + with self.assertRaises(ConfigError) as cm: + ApplicationServiceStore(hs) + + e = cm.exception + self.assertIn(f1, e.message) + self.assertIn(f2, e.message) + self.assertIn("as_token", e.message) -- cgit 1.4.1 From b84d59c5f01914fe53d2673c5c7e372f5c61d088 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Feb 2016 16:22:35 +0000 Subject: Add descriptions --- synapse/storage/appservice.py | 3 ++- synapse/storage/keys.py | 1 + synapse/storage/registration.py | 1 + synapse/storage/stream.py | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) (limited to 'synapse/storage/appservice.py') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index b5aa55c0a3..1100c67714 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -276,7 +276,8 @@ class ApplicationServiceTransactionStore(SQLBaseStore): "application_services_state", dict(as_id=service.id), ["state"], - allow_none=True + allow_none=True, + desc="get_appservice_state", ) if result: defer.returnValue(result.get("state")) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 8022b8cfc6..fd05bfe54e 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -39,6 +39,7 @@ class KeyStore(SQLBaseStore): table="server_tls_certificates", keyvalues={"server_name": server_name}, retcols=("tls_certificate",), + desc="get_server_certificate", ) tls_certificate = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_ASN1, tls_certificate_bytes, diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 70cde0d04d..bd35e19be6 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -134,6 +134,7 @@ class RegistrationStore(SQLBaseStore): }, retcols=["name", "password_hash", "is_guest"], allow_none=True, + desc="get_user_by_id", ) def get_users_by_id_case_insensitive(self, user_id): diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 2c49a5e499..50436cb2d2 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -564,6 +564,7 @@ class StreamStore(SQLBaseStore): table="events", keyvalues={"event_id": event_id}, retcols=("stream_ordering", "topological_ordering"), + desc="get_topological_token_for_event", ).addCallback(lambda row: "t%d-%d" % ( row["topological_ordering"], row["stream_ordering"],) ) -- cgit 1.4.1