From 94a5db9f4d400a345c5d8b9f7bacb0c9ccf99959 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 14:44:16 +0000 Subject: Add appservice package and move ApplicationService into it. --- synapse/appservice/__init__.py | 119 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 synapse/appservice/__init__.py (limited to 'synapse/appservice/__init__.py') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py new file mode 100644 index 0000000000..f801fb5324 --- /dev/null +++ b/synapse/appservice/__init__.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.api.constants import EventTypes + +import re + + +class ApplicationService(object): + """Defines an application service. This definition is mostly what is + provided to the /register AS API. + + Provides methods to check if this service is "interested" in events. + """ + NS_USERS = "users" + NS_ALIASES = "aliases" + NS_ROOMS = "rooms" + # The ordering here is important as it is used to map database values (which + # are stored as ints representing the position in this list) to namespace + # values. + NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] + + def __init__(self, token, url=None, namespaces=None): + self.token = token + self.url = url + self.namespaces = self._check_namespaces(namespaces) + + def _check_namespaces(self, namespaces): + # Sanity check that it is of the form: + # { + # users: ["regex",...], + # aliases: ["regex",...], + # rooms: ["regex",...], + # } + if not namespaces: + return None + + for ns in ApplicationService.NS_LIST: + if type(namespaces[ns]) != list: + raise ValueError("Bad namespace value for '%s'", ns) + for regex in namespaces[ns]: + if not isinstance(regex, basestring): + raise ValueError("Expected string regex for ns '%s'", ns) + return namespaces + + def _matches_regex(self, test_string, namespace_key): + for regex in self.namespaces[namespace_key]: + if re.match(regex, test_string): + return True + return False + + def _matches_user(self, event): + if (hasattr(event, "user_id") and + self._matches_regex( + event.user_id, ApplicationService.NS_USERS)): + return True + # also check m.room.member state key + if (hasattr(event, "type") and event.type == EventTypes.Member + and hasattr(event, "state_key") + and self._matches_regex( + event.state_key, ApplicationService.NS_USERS)): + return True + return False + + def _matches_room_id(self, event): + if hasattr(event, "room_id"): + return self._matches_regex( + event.room_id, ApplicationService.NS_ROOMS + ) + return False + + def _matches_aliases(self, event, alias_list): + for alias in alias_list: + if self._matches_regex(alias, ApplicationService.NS_ALIASES): + return True + return False + + def is_interested(self, event, restrict_to=None, aliases_for_event=None): + """Check if this service is interested in this event. + + Args: + event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. + aliases_for_event(list): A list of all the known room aliases for + this event. + Returns: + bool: True if this service would like to know about this event. + """ + if aliases_for_event is None: + aliases_for_event = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so fail early and raise a general + # exception + raise Exception("Unexpected restrict_to value: %s". restrict_to) + + if not restrict_to: + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) + elif restrict_to == ApplicationService.NS_ALIASES: + return self._matches_aliases(event, aliases_for_event) + elif restrict_to == ApplicationService.NS_ROOMS: + return self._matches_room_id(event) + elif restrict_to == ApplicationService.NS_USERS: + return self._matches_user(event) + + def __str__(self): + return "ApplicationService: %s" % (self.__dict__,) -- cgit 1.5.1 From 17753f0c20d0d8190095c5a3183630b78bf9650c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 11:19:18 +0000 Subject: Add stub ApplicationServiceApi and glue it with the handler. --- synapse/appservice/__init__.py | 3 ++- synapse/appservice/api.py | 21 +++++++++++++++++++++ synapse/handlers/appservice.py | 18 +++++++++++++++--- synapse/storage/appservice.py | 1 + 4 files changed, 39 insertions(+), 4 deletions(-) (limited to 'synapse/appservice/__init__.py') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f801fb5324..92f64619c9 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -31,10 +31,11 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - def __init__(self, token, url=None, namespaces=None): + def __init__(self, token, url=None, namespaces=None, txn_id=None): self.token = token self.url = url self.namespaces = self._check_namespaces(namespaces) + self.txn_id = None def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 803f97ea4f..158aded66e 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,3 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. + +class ApplicationServiceApi(object): + """This class manages HS -> AS communications, including querying and + pushing. + """ + + def __init__(self, hs): + self.hs_token = "_hs_token_" # TODO extract hs token + + def query_user(self, service, user_id): + pass + + def query_alias(self, service, alias): + pass + + def push_bulk(self, service, events): + pass + + def push(self, service, event): + pass + diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index f05b57bcb9..9cdeaa2d94 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService +from synapse.appservice.api import ApplicationServiceApi import logging @@ -29,6 +30,7 @@ class ApplicationServicesHandler(BaseHandler): def __init__(self, hs): super(ApplicationServicesHandler, self).__init__(hs) + self.appservice_api = ApplicationServiceApi(hs) @defer.inlineCallbacks def register(self, app_service): @@ -97,7 +99,12 @@ class ApplicationServicesHandler(BaseHandler): ) for user_service in user_query_services: # this needs to block XXX: Need to feed response back to caller - pass # TODO poke User Query API + is_known_user = self.appservice_api.query_user( + user_service, event + ) + if is_known_user: + # the user exists now,so don't query more ASes. + break # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. @@ -109,8 +116,13 @@ class ApplicationServicesHandler(BaseHandler): ) for alias_service in alias_query_services: # this needs to block XXX: Need to feed response back to caller - pass # TODO poke Room Alias Query API + is_known_alias = self.appservice_api.query_alias( + alias_service, event + ) + if is_known_alias: + # the alias exists now so don't query more ASes. + break # Fork off pushes to these services - XXX First cut, best effort for service in services: - pass # TODO push event to service + self.appservice_api.push(service, event) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 48bc7e0fe6..abb617f049 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -216,6 +216,7 @@ class ApplicationServiceStore(SQLBaseStore): except IndexError: logger.error("Bad namespace enum '%s'. %s", ns_int, res) + # TODO get last successful txn id f.e. service for service in services.values(): logger.info("Found application service: %s", service) self.cache.services.append(ApplicationService( -- cgit 1.5.1 From 525a218b2b072b24721c9c9efae42aae21388fc8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 12:24:20 +0000 Subject: Begin to add unit tests for appservice glue and regex testing. --- synapse/appservice/__init__.py | 14 ++++++-- synapse/handlers/appservice.py | 8 +++-- tests/appservice/__init__.py | 14 ++++++++ tests/appservice/test_appservice.py | 58 +++++++++++++++++++++++++++++++ tests/handlers/test_appservice.py | 68 +++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 tests/appservice/__init__.py create mode 100644 tests/appservice/test_appservice.py create mode 100644 tests/handlers/test_appservice.py (limited to 'synapse/appservice/__init__.py') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 92f64619c9..0c7f58574e 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -14,8 +14,11 @@ # limitations under the License. from synapse.api.constants import EventTypes +import logging import re +logger = logging.getLogger(__name__) + class ApplicationService(object): """Defines an application service. This definition is mostly what is @@ -56,15 +59,22 @@ class ApplicationService(object): return namespaces def _matches_regex(self, test_string, namespace_key): + if not isinstance(test_string, basestring): + logger.warning( + "Expected a string to test regex against, but got %s", + test_string + ) + return False + for regex in self.namespaces[namespace_key]: if re.match(regex, test_string): return True return False def _matches_user(self, event): - if (hasattr(event, "user_id") and + if (hasattr(event, "sender") and self._matches_regex( - event.user_id, ApplicationService.NS_USERS)): + event.sender, ApplicationService.NS_USERS)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 9cdeaa2d94..3188c60f3d 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -26,10 +26,14 @@ import logging logger = logging.getLogger(__name__) -class ApplicationServicesHandler(BaseHandler): +# NB: Purposefully not inheriting BaseHandler since that contains way too much +# setup code which this handler does not need or use. This makes testing a lot +# easier. +class ApplicationServicesHandler(object): def __init__(self, hs): - super(ApplicationServicesHandler, self).__init__(hs) + self.store = hs.get_datastore() + self.hs = hs self.appservice_api = ApplicationServiceApi(hs) @defer.inlineCallbacks diff --git a/tests/appservice/__init__.py b/tests/appservice/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/tests/appservice/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py new file mode 100644 index 0000000000..5cfd26daa6 --- /dev/null +++ b/tests/appservice/test_appservice.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.appservice import ApplicationService + +from mock import Mock, PropertyMock +from tests import unittest + + +class ApplicationServiceTestCase(unittest.TestCase): + + def setUp(self): + self.service = ApplicationService( + url="some_url", + token="some_token", + namespaces={ + ApplicationService.NS_USERS: [], + ApplicationService.NS_ROOMS: [], + ApplicationService.NS_ALIASES: [] + } + ) + self.event = Mock( + type="m.something", room_id="!foo:bar", sender="@someone:somewhere" + ) + + def test_regex_user_id_prefix_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_user_id_prefix_no_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.assertFalse(self.service.is_interested(self.event)) + + def test_regex_room_member_is_checked(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.event.type = "m.room.member" + self.event.state_key = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py new file mode 100644 index 0000000000..9c464e7fbc --- /dev/null +++ b/tests/handlers/test_appservice.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer +from .. import unittest + +from synapse.handlers.appservice import ApplicationServicesHandler + +from collections import namedtuple +from mock import Mock + +# TODO: Should this be a more general thing? tests/api/test_filtering.py uses it +MockEvent = namedtuple("MockEvent", "sender type room_id") + + +class AppServiceHandlerTestCase(unittest.TestCase): + """ Tests the ApplicationServicesHandler. """ + + def setUp(self): + self.mock_store = Mock() + self.mock_as_api = Mock() + hs = Mock() + hs.get_datastore = Mock(return_value=self.mock_store) + self.handler = ApplicationServicesHandler(hs) # thing being tested + + # FIXME Would be nice to DI this rather than monkey patch:( + if not hasattr(self.handler, "appservice_api"): + # someone probably updated the handler but not the tests. Fail fast. + raise Exception("Test expected handler.appservice_api to exist.") + self.handler.appservice_api = self.mock_as_api + + def test_notify_interested_services(self): + interested_service = self._mkservice(is_interested=True) + services = [ + self._mkservice(is_interested=False), + interested_service, + self._mkservice(is_interested=False) + ] + + self.mock_store.get_app_services = Mock(return_value=services) + + event = MockEvent( + sender="@someone:anywhere", + type="m.room.message", + room_id="!foo:bar" + ) + self.mock_as_api.push = Mock() + self.handler.notify_interested_services(event) + self.mock_as_api.push.assert_called_once_with(interested_service, event) + + def _mkservice(self, is_interested): + service = Mock() + service.is_interested = Mock(return_value=is_interested) + service.token = "mock_service_token" + service.url = "mock_service_url" + return service -- cgit 1.5.1 From 27091f146a0ebdbfe1ae7c5cd30de51515cfbebc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 10:08:12 +0000 Subject: Add hs_token column and generate a different token f.e application service. --- synapse/appservice/__init__.py | 6 ++++-- synapse/appservice/api.py | 8 ++++---- synapse/handlers/appservice.py | 9 ++++++--- synapse/rest/appservice/v1/register.py | 4 ++-- synapse/storage/appservice.py | 17 ++++++++++++----- synapse/storage/schema/application_services.sql | 1 + tests/storage/test_appservice.py | 10 ++++++---- 7 files changed, 35 insertions(+), 20 deletions(-) (limited to 'synapse/appservice/__init__.py') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 0c7f58574e..f7baf578f0 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -34,11 +34,13 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - def __init__(self, token, url=None, namespaces=None, txn_id=None): + def __init__(self, token, url=None, namespaces=None, hs_token=None, + txn_id=None): self.token = token self.url = url + self.hs_token = hs_token self.namespaces = self._check_namespaces(namespaces) - self.txn_id = None + self.txn_id = txn_id def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index fbf4abc526..29bb35d61b 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -30,7 +30,6 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) - self.hs_token = "_hs_token_" # TODO extract hs token @defer.inlineCallbacks def query_user(self, service, user_id): @@ -38,7 +37,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.get_json(uri, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object defer.returnValue(True) @@ -54,7 +53,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.get_json(uri, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object defer.returnValue(True) @@ -76,9 +75,10 @@ class ApplicationServiceApi(SimpleHttpClient): "events": events }, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object + # TODO: Mark txn as sent successfully defer.returnValue(True) except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2b2761682f..7b0599c71e 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -19,6 +19,7 @@ from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi +import synapse.util.stringutils as stringutils import logging @@ -53,10 +54,9 @@ class ApplicationServicesHandler(object): errcode=Codes.FORBIDDEN ) logger.info("Updating application service info...") + app_service.hs_token = self._generate_hs_token() yield self.store.update_app_service(app_service) - - logger.info("Sending ping to %s...", app_service.url) - yield self.appservice_api.push(app_service, "pinger") + defer.returnValue(app_service) def unregister(self, token): logger.info("Unregister as_token=%s", token) @@ -136,3 +136,6 @@ class ApplicationServicesHandler(object): # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + + def _generate_hs_token(self): + return stringutils.random_string(18) diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index e374d538e7..d3d5aef220 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -61,8 +61,8 @@ class RegisterRestServlet(AppServiceRestServlet): app_service = ApplicationService(as_token, as_url, namespaces) - yield self.handler.register(app_service) - hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? + app_service = yield self.handler.register(app_service) + hs_token = app_service.hs_token defer.returnValue((200, { "hs_token": hs_token diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index b64416de28..3c8bf9ad0d 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -60,6 +60,7 @@ class ApplicationServiceStore(SQLBaseStore): if service.token == token: service.url = None service.namespaces = None + service.hs_token = None def _unregister_app_service_txn(self, txn, token): # kill the url to prevent pushes @@ -100,6 +101,9 @@ class ApplicationServiceStore(SQLBaseStore): if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") + if not service.hs_token: + raise StoreError(500, "No HS token") + yield self.runInteraction( "update_app_service", self._update_app_service_txn, @@ -126,8 +130,8 @@ class ApplicationServiceStore(SQLBaseStore): return False txn.execute( - "UPDATE application_services SET url=? WHERE id=?", - (service.url, as_id,) + "UPDATE application_services SET url=?, hs_token=? WHERE id=?", + (service.url, service.hs_token, as_id,) ) # cleanup regex txn.execute( @@ -196,6 +200,7 @@ class ApplicationServiceStore(SQLBaseStore): # 'namespace': enum, # 'as_id': 0, # 'token': "something", + # 'hs_token': "otherthing", # 'id': 0 # } # ] @@ -208,6 +213,7 @@ class ApplicationServiceStore(SQLBaseStore): services[as_token] = { "url": res["url"], "token": as_token, + "hs_token": res["hs_token"], "namespaces": { ApplicationService.NS_USERS: [], ApplicationService.NS_ALIASES: [], @@ -230,8 +236,9 @@ class ApplicationServiceStore(SQLBaseStore): for service in services.values(): logger.info("Found application service: %s", service) self.cache.services.append(ApplicationService( - service["token"], - service["url"], - service["namespaces"] + token=service["token"], + url=service["url"], + namespaces=service["namespaces"], + hs_token=service["hs_token"] )) diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql index 6d245fc807..03b5a10c8a 100644 --- a/synapse/storage/schema/application_services.sql +++ b/synapse/storage/schema/application_services.sql @@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS application_services( id INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT, token TEXT, + hs_token TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 56fdda377c..b9ecfb3384 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -46,13 +46,15 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_update_and_retrieval_of_service(self): url = "https://matrix.org/appservices/foobar" + hs_token = "hstok" user_regex = ["@foobar_.*:matrix.org"] alias_regex = ["#foobar_.*:matrix.org"] room_regex = [] - service = ApplicationService(url=url, token=self.as_token, namespaces={ - ApplicationService.NS_USERS: user_regex, - ApplicationService.NS_ALIASES: alias_regex, - ApplicationService.NS_ROOMS: room_regex + service = ApplicationService( + url=url, hs_token=hs_token, token=self.as_token, namespaces={ + ApplicationService.NS_USERS: user_regex, + ApplicationService.NS_ALIASES: alias_regex, + ApplicationService.NS_ROOMS: room_regex }) yield self.store.update_app_service(service) -- cgit 1.5.1 From 51d63ac329a2613c0a7195e6183d70f789b7d823 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 13:19:46 +0000 Subject: Glue AS work to general event notifications. Add more exception handling when poking ASes. --- synapse/appservice/__init__.py | 2 +- synapse/appservice/api.py | 11 ++++++++++- synapse/handlers/appservice.py | 39 +++++++++++++++++++++++++-------------- synapse/notifier.py | 6 ++++++ 4 files changed, 42 insertions(+), 16 deletions(-) (limited to 'synapse/appservice/__init__.py') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f7baf578f0..d9ca856c8b 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -62,7 +62,7 @@ class ApplicationService(object): def _matches_regex(self, test_string, namespace_key): if not isinstance(test_string, basestring): - logger.warning( + logger.error( "Expected a string to test regex against, but got %s", test_string ) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 29bb35d61b..d96caf7f58 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -46,6 +46,9 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) return logger.warning("query_user to %s received %s", uri, e.code) + except Exception as ex: + logger.warning("query_user to %s threw exception %s", uri, ex) + defer.returnValue(False) @defer.inlineCallbacks def query_alias(self, service, alias): @@ -62,6 +65,10 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) return logger.warning("query_alias to %s received %s", uri, e.code) + except Exception as ex: + logger.warning("query_alias to %s threw exception %s", uri, ex) + defer.returnValue(False) + @defer.inlineCallbacks def push_bulk(self, service, events): @@ -82,7 +89,9 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(True) except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) - defer.returnValue(False) + except Exception as ex: + logger.warning("push_bulk to %s threw exception %s", uri, ex) + defer.returnValue(False) @defer.inlineCallbacks def push(self, service, event): diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2c6d4e2815..ef94215133 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -23,7 +23,6 @@ import synapse.util.stringutils as stringutils import logging - logger = logging.getLogger(__name__) @@ -58,6 +57,7 @@ class ApplicationServicesHandler(object): yield self.store.update_app_service(app_service) defer.returnValue(app_service) + @defer.inlineCallbacks def unregister(self, token): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) @@ -81,34 +81,45 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: - user_query_services = yield self._get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_USERS - ) - for user_service in user_query_services: - # this needs to block XXX: Need to feed response back to caller - is_known_user = yield self.appservice_api.query_user( - user_service, event.sender - ) - if is_known_user: - # the user exists now,so don't query more ASes. - break + yield self.query_user_exists(event) # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + @defer.inlineCallbacks + def query_user_exists(self, event): + """Check if an application services knows this event.sender exists. + + Args: + event: An event sent by the user to query + Returns: + True if this user exists. + """ + # TODO Would be nice for this to accept a user ID instead of an event. + user_query_services = yield self._get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_USERS + ) + for user_service in user_query_services: + is_known_user = yield self.appservice_api.query_user( + user_service, event.sender + ) + if is_known_user: + defer.returnValue(True) + defer.returnValue(False) @defer.inlineCallbacks def query_room_alias_exists(self, room_alias): """Check if an application service knows this room alias exists. Args: - room_alias(str): The room alias to query. + room_alias(RoomAlias): The room alias to query. Returns: namedtuple: with keys "room_id" and "servers" or None if no association can be found. """ + room_alias = room_alias.to_string() alias_query_services = yield self._get_services_for_event( event=None, restrict_to=ApplicationService.NS_ALIASES, diff --git a/synapse/notifier.py b/synapse/notifier.py index e3b6ead620..c7f75ab801 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -99,6 +99,12 @@ class Notifier(object): `extra_users` param. """ yield run_on_reactor() + + # poke any interested application service. + self.hs.get_handlers().appservice_handler.notify_interested_services( + event + ) + room_id = event.room_id room_source = self.event_sources.sources["room"] -- cgit 1.5.1 From c71456117dc70dfe0bfa15e3f655a3ac1dfc66ee Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 14:17:08 +0000 Subject: Fix user query checks. HS>AS pushing now works. --- synapse/appservice/__init__.py | 21 ++++++++++++-------- synapse/handlers/appservice.py | 44 +++++++++++++++++++++++++++++------------- 2 files changed, 44 insertions(+), 21 deletions(-) (limited to 'synapse/appservice/__init__.py') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index d9ca856c8b..46d46a5a48 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -75,27 +75,23 @@ class ApplicationService(object): def _matches_user(self, event): if (hasattr(event, "sender") and - self._matches_regex( - event.sender, ApplicationService.NS_USERS)): + self.is_interested_in_user(event.sender)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member and hasattr(event, "state_key") - and self._matches_regex( - event.state_key, ApplicationService.NS_USERS)): + and self.is_interested_in_user(event.state_key)): return True return False def _matches_room_id(self, event): if hasattr(event, "room_id"): - return self._matches_regex( - event.room_id, ApplicationService.NS_ROOMS - ) + return self.is_interested_in_room(event.room_id) return False def _matches_aliases(self, event, alias_list): for alias in alias_list: - if self._matches_regex(alias, ApplicationService.NS_ALIASES): + if self.is_interested_in_alias(alias): return True return False @@ -128,5 +124,14 @@ class ApplicationService(object): elif restrict_to == ApplicationService.NS_USERS: return self._matches_user(event) + def is_interested_in_user(self, user_id): + return self._matches_regex(user_id, ApplicationService.NS_USERS) + + def is_interested_in_alias(self, alias): + return self._matches_regex(alias, ApplicationService.NS_ALIASES) + + def is_interested_in_room(self, room_id): + return self._matches_regex(room_id, ApplicationService.NS_ROOMS) + def __str__(self): return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index ef94215133..8d0cdd528c 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,6 +15,7 @@ from twisted.internet import defer +from synapse.api.constants import EventTypes from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi @@ -78,32 +79,31 @@ class ApplicationServicesHandler(object): return # no services need notifying # Do we know this user exists? If not, poke the user query API for - # all services which match that user regex. - unknown_user = yield self._is_unknown_user(event.sender) - if unknown_user: - yield self.query_user_exists(event) + # all services which match that user regex. This needs to block as these + # user queries need to be made BEFORE pushing the event. + yield self._check_user_exists(event.sender) + if event.type == EventTypes.Member: + yield self._check_user_exists(event.state_key) # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) @defer.inlineCallbacks - def query_user_exists(self, event): - """Check if an application services knows this event.sender exists. + def query_user_exists(self, user_id): + """Check if any application service knows this user_id exists. Args: - event: An event sent by the user to query + user_id(str): The user to query if they exist on any AS. Returns: - True if this user exists. + True if this user exists on at least one application service. """ - # TODO Would be nice for this to accept a user ID instead of an event. - user_query_services = yield self._get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_USERS + user_query_services = yield self._get_services_for_user( + user_id=user_id ) for user_service in user_query_services: is_known_user = yield self.appservice_api.query_user( - user_service, event.sender + user_service, user_id ) if is_known_user: defer.returnValue(True) @@ -161,6 +161,16 @@ class ApplicationServicesHandler(object): ] defer.returnValue(interested_list) + @defer.inlineCallbacks + def _get_services_for_user(self, user_id): + services = yield self.store.get_app_services() + interested_list = [ + s for s in services if ( + s.is_interested_in_user(user_id) + ) + ] + defer.returnValue(interested_list) + @defer.inlineCallbacks def _is_unknown_user(self, user_id): user = UserID.from_string(user_id) @@ -173,5 +183,13 @@ class ApplicationServicesHandler(object): user_info = yield self.store.get_user_by_id(user_id) defer.returnValue(len(user_info) == 0) + @defer.inlineCallbacks + def _check_user_exists(self, user_id): + unknown_user = yield self._is_unknown_user(user_id) + if unknown_user: + exists = yield self.query_user_exists(user_id) + defer.returnValue(exists) + defer.returnValue(True) + def _generate_hs_token(self): return stringutils.random_string(24) -- cgit 1.5.1 From ac3183caaa66b750996d90c0ac9ed430f623909c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 12:03:37 +0000 Subject: Register a user account for the AS when the AS registers. Add 'sender' column to AS table. --- synapse/appservice/__init__.py | 3 ++- synapse/handlers/appservice.py | 8 +++++++- synapse/storage/appservice.py | 5 +++-- synapse/storage/schema/application_services.sql | 1 + synapse/storage/schema/delta/v14.sql | 1 + 5 files changed, 14 insertions(+), 4 deletions(-) (limited to 'synapse/appservice/__init__.py') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 46d46a5a48..fb9bfffe5d 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -35,10 +35,11 @@ class ApplicationService(object): NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None, hs_token=None, - txn_id=None): + sender=None, txn_id=None): self.token = token self.url = url self.hs_token = hs_token + self.sender = sender self.namespaces = self._check_namespaces(namespaces) self.txn_id = txn_id diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index fa810b9a98..5071a12eb1 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -52,8 +52,14 @@ class ApplicationServicesHandler(object): "Consult the home server admin.", errcode=Codes.FORBIDDEN ) - logger.info("Updating application service info...") + app_service.hs_token = self._generate_hs_token() + + # create a sender for this application service which is used when + # creating rooms, etc.. + account = yield self.hs.get_handlers().registration_handler.register() + app_service.sender = account[0] + yield self.store.update_app_service(app_service) defer.returnValue(app_service) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 3c8bf9ad0d..eef77e737e 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -130,8 +130,9 @@ class ApplicationServiceStore(SQLBaseStore): return False txn.execute( - "UPDATE application_services SET url=?, hs_token=? WHERE id=?", - (service.url, service.hs_token, as_id,) + "UPDATE application_services SET url=?, hs_token=?, sender=? " + "WHERE id=?", + (service.url, service.hs_token, service.sender, as_id,) ) # cleanup regex txn.execute( diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql index 03b5a10c8a..e491ad5aec 100644 --- a/synapse/storage/schema/application_services.sql +++ b/synapse/storage/schema/application_services.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS application_services( url TEXT, token TEXT, hs_token TEXT, + sender TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); diff --git a/synapse/storage/schema/delta/v14.sql b/synapse/storage/schema/delta/v14.sql index 03b5a10c8a..e491ad5aec 100644 --- a/synapse/storage/schema/delta/v14.sql +++ b/synapse/storage/schema/delta/v14.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS application_services( url TEXT, token TEXT, hs_token TEXT, + sender TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); -- cgit 1.5.1 From c7783d6feec9b69c24f3303cbb51cce3e6b8ffb3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:36:08 +0000 Subject: Notify ASes for events sent by other users in a room which an AS user is a part of. --- synapse/appservice/__init__.py | 17 +++++++++++++---- synapse/handlers/appservice.py | 21 ++++++++++++++++----- synapse/rest/appservice/v1/__init__.py | 4 ++-- tests/appservice/test_appservice.py | 25 +++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 11 deletions(-) (limited to 'synapse/appservice/__init__.py') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index fb9bfffe5d..381b4cfc4a 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -74,7 +74,7 @@ class ApplicationService(object): return True return False - def _matches_user(self, event): + def _matches_user(self, event, member_list): if (hasattr(event, "sender") and self.is_interested_in_user(event.sender)): return True @@ -83,6 +83,10 @@ class ApplicationService(object): and hasattr(event, "state_key") and self.is_interested_in_user(event.state_key)): return True + # check joined member events + for member in member_list: + if self.is_interested_in_user(member.state_key): + return True return False def _matches_room_id(self, event): @@ -96,7 +100,8 @@ class ApplicationService(object): return True return False - def is_interested(self, event, restrict_to=None, aliases_for_event=None): + def is_interested(self, event, restrict_to=None, aliases_for_event=None, + member_list=None): """Check if this service is interested in this event. Args: @@ -104,18 +109,22 @@ class ApplicationService(object): restrict_to(str): The namespace to restrict regex tests to. aliases_for_event(list): A list of all the known room aliases for this event. + member_list(list): A list of all joined room members in this room. Returns: bool: True if this service would like to know about this event. """ if aliases_for_event is None: aliases_for_event = [] + if member_list is None: + member_list = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: # this is a programming error, so fail early and raise a general # exception raise Exception("Unexpected restrict_to value: %s". restrict_to) if not restrict_to: - return (self._matches_user(event) + return (self._matches_user(event, member_list) or self._matches_aliases(event, aliases_for_event) or self._matches_room_id(event)) elif restrict_to == ApplicationService.NS_ALIASES: @@ -123,7 +132,7 @@ class ApplicationService(object): elif restrict_to == ApplicationService.NS_ROOMS: return self._matches_room_id(event) elif restrict_to == ApplicationService.NS_USERS: - return self._matches_user(event) + return self._matches_user(event, member_list) def is_interested_in_user(self, user_id): return self._matches_regex(user_id, ApplicationService.NS_USERS) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8591a77bf3..2c488a46f6 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, Membership from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.types import UserID @@ -154,14 +154,25 @@ class ApplicationServicesHandler(object): list: A list of services interested in this event based on the service regex. """ - # We need to know the aliases associated with this event.room_id, if any - if not alias_list: - alias_list = yield self.store.get_aliases_for_room(event.room_id) + member_list = None + if hasattr(event, "room_id"): + # We need to know the aliases associated with this event.room_id, + # if any. + if not alias_list: + alias_list = yield self.store.get_aliases_for_room( + event.room_id + ) + # We need to know the members associated with this event.room_id, + # if any. + member_list = yield self.store.get_room_members( + room_id=event.room_id, + membership=Membership.JOIN + ) services = yield self.store.get_app_services() interested_list = [ s for s in services if ( - s.is_interested(event, restrict_to, alias_list) + s.is_interested(event, restrict_to, alias_list, member_list) ) ] defer.returnValue(interested_list) diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py index bf243b6180..a7877609ad 100644 --- a/synapse/rest/appservice/v1/__init__.py +++ b/synapse/rest/appservice/v1/__init__.py @@ -21,9 +21,9 @@ class AppServiceRestResource(JsonResource): """A resource for version 1 of the matrix application service API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod def register_servlets(appservice_resource, hs): - register.register_servlets(hs, appservice_resource) \ No newline at end of file + register.register_servlets(hs, appservice_resource) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index c0aaf12785..d12e4f2644 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -143,3 +143,28 @@ class ApplicationServiceTestCase(unittest.TestCase): restrict_to=ApplicationService.NS_USERS, aliases_for_event=["#xmpp_barfoo:matrix.org"] )) + + def test_member_list_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + join_list = [ + Mock( + type="m.room.member", room_id="!foo:bar", sender="@alice:here", + state_key="@alice:here" + ), + Mock( + type="m.room.member", room_id="!foo:bar", sender="@irc_fo:here", + state_key="@irc_fo:here" # AS user + ), + Mock( + type="m.room.member", room_id="!foo:bar", sender="@bob:here", + state_key="@bob:here" + ) + ] + + self.event.sender = "@xmpp_foobar:matrix.org" + self.assertTrue(self.service.is_interested( + event=self.event, + member_list=join_list + )) -- cgit 1.5.1