From 94a5db9f4d400a345c5d8b9f7bacb0c9ccf99959 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 14:44:16 +0000 Subject: Add appservice package and move ApplicationService into it. --- synapse/appservice/__init__.py | 119 +++++++++++++++++++++++++++++++++++++++++ synapse/appservice/api.py | 15 ++++++ 2 files changed, 134 insertions(+) create mode 100644 synapse/appservice/__init__.py create mode 100644 synapse/appservice/api.py (limited to 'synapse/appservice') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py new file mode 100644 index 0000000000..f801fb5324 --- /dev/null +++ b/synapse/appservice/__init__.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.api.constants import EventTypes + +import re + + +class ApplicationService(object): + """Defines an application service. This definition is mostly what is + provided to the /register AS API. + + Provides methods to check if this service is "interested" in events. + """ + NS_USERS = "users" + NS_ALIASES = "aliases" + NS_ROOMS = "rooms" + # The ordering here is important as it is used to map database values (which + # are stored as ints representing the position in this list) to namespace + # values. + NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] + + def __init__(self, token, url=None, namespaces=None): + self.token = token + self.url = url + self.namespaces = self._check_namespaces(namespaces) + + def _check_namespaces(self, namespaces): + # Sanity check that it is of the form: + # { + # users: ["regex",...], + # aliases: ["regex",...], + # rooms: ["regex",...], + # } + if not namespaces: + return None + + for ns in ApplicationService.NS_LIST: + if type(namespaces[ns]) != list: + raise ValueError("Bad namespace value for '%s'", ns) + for regex in namespaces[ns]: + if not isinstance(regex, basestring): + raise ValueError("Expected string regex for ns '%s'", ns) + return namespaces + + def _matches_regex(self, test_string, namespace_key): + for regex in self.namespaces[namespace_key]: + if re.match(regex, test_string): + return True + return False + + def _matches_user(self, event): + if (hasattr(event, "user_id") and + self._matches_regex( + event.user_id, ApplicationService.NS_USERS)): + return True + # also check m.room.member state key + if (hasattr(event, "type") and event.type == EventTypes.Member + and hasattr(event, "state_key") + and self._matches_regex( + event.state_key, ApplicationService.NS_USERS)): + return True + return False + + def _matches_room_id(self, event): + if hasattr(event, "room_id"): + return self._matches_regex( + event.room_id, ApplicationService.NS_ROOMS + ) + return False + + def _matches_aliases(self, event, alias_list): + for alias in alias_list: + if self._matches_regex(alias, ApplicationService.NS_ALIASES): + return True + return False + + def is_interested(self, event, restrict_to=None, aliases_for_event=None): + """Check if this service is interested in this event. + + Args: + event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. + aliases_for_event(list): A list of all the known room aliases for + this event. + Returns: + bool: True if this service would like to know about this event. + """ + if aliases_for_event is None: + aliases_for_event = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so fail early and raise a general + # exception + raise Exception("Unexpected restrict_to value: %s". restrict_to) + + if not restrict_to: + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) + elif restrict_to == ApplicationService.NS_ALIASES: + return self._matches_aliases(event, aliases_for_event) + elif restrict_to == ApplicationService.NS_ROOMS: + return self._matches_room_id(event) + elif restrict_to == ApplicationService.NS_USERS: + return self._matches_user(event) + + def __str__(self): + return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py new file mode 100644 index 0000000000..803f97ea4f --- /dev/null +++ b/synapse/appservice/api.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- cgit 1.5.1 From 17753f0c20d0d8190095c5a3183630b78bf9650c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 11:19:18 +0000 Subject: Add stub ApplicationServiceApi and glue it with the handler. --- synapse/appservice/__init__.py | 3 ++- synapse/appservice/api.py | 21 +++++++++++++++++++++ synapse/handlers/appservice.py | 18 +++++++++++++++--- synapse/storage/appservice.py | 1 + 4 files changed, 39 insertions(+), 4 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f801fb5324..92f64619c9 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -31,10 +31,11 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - def __init__(self, token, url=None, namespaces=None): + def __init__(self, token, url=None, namespaces=None, txn_id=None): self.token = token self.url = url self.namespaces = self._check_namespaces(namespaces) + self.txn_id = None def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 803f97ea4f..158aded66e 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,3 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. + +class ApplicationServiceApi(object): + """This class manages HS -> AS communications, including querying and + pushing. + """ + + def __init__(self, hs): + self.hs_token = "_hs_token_" # TODO extract hs token + + def query_user(self, service, user_id): + pass + + def query_alias(self, service, alias): + pass + + def push_bulk(self, service, events): + pass + + def push(self, service, event): + pass + diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index f05b57bcb9..9cdeaa2d94 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService +from synapse.appservice.api import ApplicationServiceApi import logging @@ -29,6 +30,7 @@ class ApplicationServicesHandler(BaseHandler): def __init__(self, hs): super(ApplicationServicesHandler, self).__init__(hs) + self.appservice_api = ApplicationServiceApi(hs) @defer.inlineCallbacks def register(self, app_service): @@ -97,7 +99,12 @@ class ApplicationServicesHandler(BaseHandler): ) for user_service in user_query_services: # this needs to block XXX: Need to feed response back to caller - pass # TODO poke User Query API + is_known_user = self.appservice_api.query_user( + user_service, event + ) + if is_known_user: + # the user exists now,so don't query more ASes. + break # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. @@ -109,8 +116,13 @@ class ApplicationServicesHandler(BaseHandler): ) for alias_service in alias_query_services: # this needs to block XXX: Need to feed response back to caller - pass # TODO poke Room Alias Query API + is_known_alias = self.appservice_api.query_alias( + alias_service, event + ) + if is_known_alias: + # the alias exists now so don't query more ASes. + break # Fork off pushes to these services - XXX First cut, best effort for service in services: - pass # TODO push event to service + self.appservice_api.push(service, event) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 48bc7e0fe6..abb617f049 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -216,6 +216,7 @@ class ApplicationServiceStore(SQLBaseStore): except IndexError: logger.error("Bad namespace enum '%s'. %s", ns_int, res) + # TODO get last successful txn id f.e. service for service in services.values(): logger.info("Found application service: %s", service) self.cache.services.append(ApplicationService( -- cgit 1.5.1 From 525a218b2b072b24721c9c9efae42aae21388fc8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 12:24:20 +0000 Subject: Begin to add unit tests for appservice glue and regex testing. --- synapse/appservice/__init__.py | 14 ++++++-- synapse/handlers/appservice.py | 8 +++-- tests/appservice/__init__.py | 14 ++++++++ tests/appservice/test_appservice.py | 58 +++++++++++++++++++++++++++++++ tests/handlers/test_appservice.py | 68 +++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 tests/appservice/__init__.py create mode 100644 tests/appservice/test_appservice.py create mode 100644 tests/handlers/test_appservice.py (limited to 'synapse/appservice') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 92f64619c9..0c7f58574e 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -14,8 +14,11 @@ # limitations under the License. from synapse.api.constants import EventTypes +import logging import re +logger = logging.getLogger(__name__) + class ApplicationService(object): """Defines an application service. This definition is mostly what is @@ -56,15 +59,22 @@ class ApplicationService(object): return namespaces def _matches_regex(self, test_string, namespace_key): + if not isinstance(test_string, basestring): + logger.warning( + "Expected a string to test regex against, but got %s", + test_string + ) + return False + for regex in self.namespaces[namespace_key]: if re.match(regex, test_string): return True return False def _matches_user(self, event): - if (hasattr(event, "user_id") and + if (hasattr(event, "sender") and self._matches_regex( - event.user_id, ApplicationService.NS_USERS)): + event.sender, ApplicationService.NS_USERS)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 9cdeaa2d94..3188c60f3d 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -26,10 +26,14 @@ import logging logger = logging.getLogger(__name__) -class ApplicationServicesHandler(BaseHandler): +# NB: Purposefully not inheriting BaseHandler since that contains way too much +# setup code which this handler does not need or use. This makes testing a lot +# easier. +class ApplicationServicesHandler(object): def __init__(self, hs): - super(ApplicationServicesHandler, self).__init__(hs) + self.store = hs.get_datastore() + self.hs = hs self.appservice_api = ApplicationServiceApi(hs) @defer.inlineCallbacks diff --git a/tests/appservice/__init__.py b/tests/appservice/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/tests/appservice/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py new file mode 100644 index 0000000000..5cfd26daa6 --- /dev/null +++ b/tests/appservice/test_appservice.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.appservice import ApplicationService + +from mock import Mock, PropertyMock +from tests import unittest + + +class ApplicationServiceTestCase(unittest.TestCase): + + def setUp(self): + self.service = ApplicationService( + url="some_url", + token="some_token", + namespaces={ + ApplicationService.NS_USERS: [], + ApplicationService.NS_ROOMS: [], + ApplicationService.NS_ALIASES: [] + } + ) + self.event = Mock( + type="m.something", room_id="!foo:bar", sender="@someone:somewhere" + ) + + def test_regex_user_id_prefix_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_user_id_prefix_no_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.assertFalse(self.service.is_interested(self.event)) + + def test_regex_room_member_is_checked(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.event.type = "m.room.member" + self.event.state_key = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py new file mode 100644 index 0000000000..9c464e7fbc --- /dev/null +++ b/tests/handlers/test_appservice.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer +from .. import unittest + +from synapse.handlers.appservice import ApplicationServicesHandler + +from collections import namedtuple +from mock import Mock + +# TODO: Should this be a more general thing? tests/api/test_filtering.py uses it +MockEvent = namedtuple("MockEvent", "sender type room_id") + + +class AppServiceHandlerTestCase(unittest.TestCase): + """ Tests the ApplicationServicesHandler. """ + + def setUp(self): + self.mock_store = Mock() + self.mock_as_api = Mock() + hs = Mock() + hs.get_datastore = Mock(return_value=self.mock_store) + self.handler = ApplicationServicesHandler(hs) # thing being tested + + # FIXME Would be nice to DI this rather than monkey patch:( + if not hasattr(self.handler, "appservice_api"): + # someone probably updated the handler but not the tests. Fail fast. + raise Exception("Test expected handler.appservice_api to exist.") + self.handler.appservice_api = self.mock_as_api + + def test_notify_interested_services(self): + interested_service = self._mkservice(is_interested=True) + services = [ + self._mkservice(is_interested=False), + interested_service, + self._mkservice(is_interested=False) + ] + + self.mock_store.get_app_services = Mock(return_value=services) + + event = MockEvent( + sender="@someone:anywhere", + type="m.room.message", + room_id="!foo:bar" + ) + self.mock_as_api.push = Mock() + self.handler.notify_interested_services(event) + self.mock_as_api.push.assert_called_once_with(interested_service, event) + + def _mkservice(self, is_interested): + service = Mock() + service.is_interested = Mock(return_value=is_interested) + service.token = "mock_service_token" + service.url = "mock_service_url" + return service -- cgit 1.5.1 From aa8cce58bf3d3dbd1a0d512dbbd41b6545d2b1f9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 16:44:53 +0000 Subject: Add query_user/alias APIs. --- synapse/appservice/api.py | 47 ++++++++++++++++++++++++++++++++++++++---- synapse/handlers/appservice.py | 26 ++++++++++++++--------- 2 files changed, 59 insertions(+), 14 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 158aded66e..799ada96df 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -12,25 +12,64 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer +from twisted.web.client import PartialDownloadError +from synapse.http.client import SimpleHttpClient -class ApplicationServiceApi(object): +import logging +import urllib + +logger = logging.getLogger(__name__) + + +class ApplicationServiceApi(SimpleHttpClient): """This class manages HS -> AS communications, including querying and pushing. """ def __init__(self, hs): + super(ApplicationServiceApi, self).__init__(hs) self.hs_token = "_hs_token_" # TODO extract hs token + @defer.inlineCallbacks def query_user(self, service, user_id): - pass + uri = service.url + ("/users/%s" % urllib.quote(user_id)) + response = None + try: + response = yield self.get_json(uri, { + "access_token": self.hs_token + }) + if response: # just an empty json object + defer.returnValue(True) + except PartialDownloadError as e: + if e.status == 404: + defer.returnValue(False) + return + logger.warning("query_user to %s received %s", (uri, e.status)) + @defer.inlineCallbacks def query_alias(self, service, alias): - pass + uri = service.url + ("/rooms/%s" % urllib.quote(alias)) + response = None + try: + response = yield self.get_json(uri, { + "access_token": self.hs_token + }) + logger.info("%s", response[0]) + if response: # just an empty json object + defer.returnValue(True) + except PartialDownloadError as e: + if e.status == 404: + defer.returnValue(False) + return + logger.warning("query_alias to %s received %s", (uri, e.status)) def push_bulk(self, service, events): pass + @defer.inlineCallbacks def push(self, service, event): - pass + response = yield self.push_bulk(service, [event]) + defer.returnValue(response) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 3188c60f3d..dd860b2244 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -55,10 +55,14 @@ class ApplicationServicesHandler(object): logger.info("Updating application service info...") yield self.store.update_app_service(app_service) + logger.info("Sending ping to %s...", app_service.url) + yield self.appservice_api.query_alias(app_service, "ping") + def unregister(self, token): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) + @defer.inlineCallbacks def get_services_for_event(self, event, restrict_to=""): """Retrieve a list of application services interested in this event. @@ -71,14 +75,15 @@ class ApplicationServicesHandler(object): """ # We need to know the aliases associated with this event.room_id, if any alias_list = [] # TODO - + services = yield self.store.get_app_services() interested_list = [ - s for s in self.store.get_app_services() if ( + s for s in services if ( s.is_interested(event, restrict_to, alias_list) ) ] - return interested_list + defer.returnValue(interested_list) + @defer.inlineCallbacks def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. @@ -89,7 +94,7 @@ class ApplicationServicesHandler(object): event(Event): The event to push out to interested services. """ # Gather interested services - services = self.get_services_for_event(event) + services = yield self.get_services_for_event(event) if len(services) == 0: return # no services need notifying @@ -97,14 +102,14 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = False # TODO check if unknown_user: - user_query_services = self.get_services_for_event( + user_query_services = yield self.get_services_for_event( event=event, restrict_to=ApplicationService.NS_USERS ) for user_service in user_query_services: # this needs to block XXX: Need to feed response back to caller - is_known_user = self.appservice_api.query_user( - user_service, event + is_known_user = yield self.appservice_api.query_user( + user_service, event.sender ) if is_known_user: # the user exists now,so don't query more ASes. @@ -114,14 +119,15 @@ class ApplicationServicesHandler(object): # API for all services which match that room alias regex. unknown_room_alias = False # TODO check if unknown_room_alias: - alias_query_services = self.get_services_for_event( + alias = "something" # TODO + alias_query_services = yield self.get_services_for_event( event=event, restrict_to=ApplicationService.NS_ALIASES ) for alias_service in alias_query_services: # this needs to block XXX: Need to feed response back to caller - is_known_alias = self.appservice_api.query_alias( - alias_service, event + is_known_alias = yield self.appservice_api.query_alias( + alias_service, alias ) if is_known_alias: # the alias exists now so don't query more ASes. -- cgit 1.5.1 From 6d3e4f4d0aad4ad9b44b28349838ff48aef39440 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:32:44 +0000 Subject: Update user/alias query APIs to use new format of SimpleHttpClient.get_json --- synapse/appservice/api.py | 15 +++++++-------- synapse/http/client.py | 3 +++ 2 files changed, 10 insertions(+), 8 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 799ada96df..74508ecddf 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from twisted.internet import defer -from twisted.web.client import PartialDownloadError +from synapse.api.errors import CodeMessageException from synapse.http.client import SimpleHttpClient import logging @@ -42,11 +42,11 @@ class ApplicationServiceApi(SimpleHttpClient): }) if response: # just an empty json object defer.returnValue(True) - except PartialDownloadError as e: - if e.status == 404: + except CodeMessageException as e: + if e.code == 404: defer.returnValue(False) return - logger.warning("query_user to %s received %s", (uri, e.status)) + logger.warning("query_user to %s received %s", uri, e.code) @defer.inlineCallbacks def query_alias(self, service, alias): @@ -56,14 +56,13 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": self.hs_token }) - logger.info("%s", response[0]) if response: # just an empty json object defer.returnValue(True) - except PartialDownloadError as e: - if e.status == 404: + except CodeMessageException as e: + if e.code == 404: defer.returnValue(False) return - logger.warning("query_alias to %s received %s", (uri, e.status)) + logger.warning("query_alias to %s received %s", uri, e.code) def push_bulk(self, service, events): pass diff --git a/synapse/http/client.py b/synapse/http/client.py index 5f4558be47..fee8c901a2 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -113,6 +113,9 @@ class SimpleHttpClient(object): if 200 <= response.code < 300: defer.returnValue(json.loads(body)) else: + # NB: This is explicitly not json.loads(body)'d because the contract + # of CodeMessageException is a *string* message. Callers can always + # load it into JSON if they want. raise CodeMessageException(response.code, body) -- cgit 1.5.1 From a1a4960baf78c1a24f76c603076c832a1947360f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 09:43:22 +0000 Subject: Impl push_bulk function --- synapse/appservice/api.py | 19 ++++++++++++++++++- synapse/handlers/appservice.py | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 74508ecddf..fbf4abc526 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -64,8 +64,25 @@ class ApplicationServiceApi(SimpleHttpClient): return logger.warning("query_alias to %s received %s", uri, e.code) + @defer.inlineCallbacks def push_bulk(self, service, events): - pass + uri = service.url + ("/transactions/%s" % + urllib.quote(str(0))) # TODO txn_ids + response = None + try: + response = yield self.put_json( + uri, + { + "events": events + }, + { + "access_token": self.hs_token + }) + if response: # just an empty json object + defer.returnValue(True) + except CodeMessageException as e: + logger.warning("push_bulk to %s received %s", uri, e.code) + defer.returnValue(False) @defer.inlineCallbacks def push(self, service, event): diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index dd860b2244..2b2761682f 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -56,7 +56,7 @@ class ApplicationServicesHandler(object): yield self.store.update_app_service(app_service) logger.info("Sending ping to %s...", app_service.url) - yield self.appservice_api.query_alias(app_service, "ping") + yield self.appservice_api.push(app_service, "pinger") def unregister(self, token): logger.info("Unregister as_token=%s", token) -- cgit 1.5.1 From 27091f146a0ebdbfe1ae7c5cd30de51515cfbebc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 10:08:12 +0000 Subject: Add hs_token column and generate a different token f.e application service. --- synapse/appservice/__init__.py | 6 ++++-- synapse/appservice/api.py | 8 ++++---- synapse/handlers/appservice.py | 9 ++++++--- synapse/rest/appservice/v1/register.py | 4 ++-- synapse/storage/appservice.py | 17 ++++++++++++----- synapse/storage/schema/application_services.sql | 1 + tests/storage/test_appservice.py | 10 ++++++---- 7 files changed, 35 insertions(+), 20 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 0c7f58574e..f7baf578f0 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -34,11 +34,13 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - def __init__(self, token, url=None, namespaces=None, txn_id=None): + def __init__(self, token, url=None, namespaces=None, hs_token=None, + txn_id=None): self.token = token self.url = url + self.hs_token = hs_token self.namespaces = self._check_namespaces(namespaces) - self.txn_id = None + self.txn_id = txn_id def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index fbf4abc526..29bb35d61b 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -30,7 +30,6 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) - self.hs_token = "_hs_token_" # TODO extract hs token @defer.inlineCallbacks def query_user(self, service, user_id): @@ -38,7 +37,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.get_json(uri, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object defer.returnValue(True) @@ -54,7 +53,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.get_json(uri, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object defer.returnValue(True) @@ -76,9 +75,10 @@ class ApplicationServiceApi(SimpleHttpClient): "events": events }, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object + # TODO: Mark txn as sent successfully defer.returnValue(True) except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2b2761682f..7b0599c71e 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -19,6 +19,7 @@ from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi +import synapse.util.stringutils as stringutils import logging @@ -53,10 +54,9 @@ class ApplicationServicesHandler(object): errcode=Codes.FORBIDDEN ) logger.info("Updating application service info...") + app_service.hs_token = self._generate_hs_token() yield self.store.update_app_service(app_service) - - logger.info("Sending ping to %s...", app_service.url) - yield self.appservice_api.push(app_service, "pinger") + defer.returnValue(app_service) def unregister(self, token): logger.info("Unregister as_token=%s", token) @@ -136,3 +136,6 @@ class ApplicationServicesHandler(object): # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + + def _generate_hs_token(self): + return stringutils.random_string(18) diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index e374d538e7..d3d5aef220 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -61,8 +61,8 @@ class RegisterRestServlet(AppServiceRestServlet): app_service = ApplicationService(as_token, as_url, namespaces) - yield self.handler.register(app_service) - hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? + app_service = yield self.handler.register(app_service) + hs_token = app_service.hs_token defer.returnValue((200, { "hs_token": hs_token diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index b64416de28..3c8bf9ad0d 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -60,6 +60,7 @@ class ApplicationServiceStore(SQLBaseStore): if service.token == token: service.url = None service.namespaces = None + service.hs_token = None def _unregister_app_service_txn(self, txn, token): # kill the url to prevent pushes @@ -100,6 +101,9 @@ class ApplicationServiceStore(SQLBaseStore): if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") + if not service.hs_token: + raise StoreError(500, "No HS token") + yield self.runInteraction( "update_app_service", self._update_app_service_txn, @@ -126,8 +130,8 @@ class ApplicationServiceStore(SQLBaseStore): return False txn.execute( - "UPDATE application_services SET url=? WHERE id=?", - (service.url, as_id,) + "UPDATE application_services SET url=?, hs_token=? WHERE id=?", + (service.url, service.hs_token, as_id,) ) # cleanup regex txn.execute( @@ -196,6 +200,7 @@ class ApplicationServiceStore(SQLBaseStore): # 'namespace': enum, # 'as_id': 0, # 'token': "something", + # 'hs_token': "otherthing", # 'id': 0 # } # ] @@ -208,6 +213,7 @@ class ApplicationServiceStore(SQLBaseStore): services[as_token] = { "url": res["url"], "token": as_token, + "hs_token": res["hs_token"], "namespaces": { ApplicationService.NS_USERS: [], ApplicationService.NS_ALIASES: [], @@ -230,8 +236,9 @@ class ApplicationServiceStore(SQLBaseStore): for service in services.values(): logger.info("Found application service: %s", service) self.cache.services.append(ApplicationService( - service["token"], - service["url"], - service["namespaces"] + token=service["token"], + url=service["url"], + namespaces=service["namespaces"], + hs_token=service["hs_token"] )) diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql index 6d245fc807..03b5a10c8a 100644 --- a/synapse/storage/schema/application_services.sql +++ b/synapse/storage/schema/application_services.sql @@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS application_services( id INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT, token TEXT, + hs_token TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 56fdda377c..b9ecfb3384 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -46,13 +46,15 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_update_and_retrieval_of_service(self): url = "https://matrix.org/appservices/foobar" + hs_token = "hstok" user_regex = ["@foobar_.*:matrix.org"] alias_regex = ["#foobar_.*:matrix.org"] room_regex = [] - service = ApplicationService(url=url, token=self.as_token, namespaces={ - ApplicationService.NS_USERS: user_regex, - ApplicationService.NS_ALIASES: alias_regex, - ApplicationService.NS_ROOMS: room_regex + service = ApplicationService( + url=url, hs_token=hs_token, token=self.as_token, namespaces={ + ApplicationService.NS_USERS: user_regex, + ApplicationService.NS_ALIASES: alias_regex, + ApplicationService.NS_ROOMS: room_regex }) yield self.store.update_app_service(service) -- cgit 1.5.1 From 51d63ac329a2613c0a7195e6183d70f789b7d823 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 13:19:46 +0000 Subject: Glue AS work to general event notifications. Add more exception handling when poking ASes. --- synapse/appservice/__init__.py | 2 +- synapse/appservice/api.py | 11 ++++++++++- synapse/handlers/appservice.py | 39 +++++++++++++++++++++++++-------------- synapse/notifier.py | 6 ++++++ 4 files changed, 42 insertions(+), 16 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f7baf578f0..d9ca856c8b 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -62,7 +62,7 @@ class ApplicationService(object): def _matches_regex(self, test_string, namespace_key): if not isinstance(test_string, basestring): - logger.warning( + logger.error( "Expected a string to test regex against, but got %s", test_string ) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 29bb35d61b..d96caf7f58 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -46,6 +46,9 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) return logger.warning("query_user to %s received %s", uri, e.code) + except Exception as ex: + logger.warning("query_user to %s threw exception %s", uri, ex) + defer.returnValue(False) @defer.inlineCallbacks def query_alias(self, service, alias): @@ -62,6 +65,10 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) return logger.warning("query_alias to %s received %s", uri, e.code) + except Exception as ex: + logger.warning("query_alias to %s threw exception %s", uri, ex) + defer.returnValue(False) + @defer.inlineCallbacks def push_bulk(self, service, events): @@ -82,7 +89,9 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(True) except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) - defer.returnValue(False) + except Exception as ex: + logger.warning("push_bulk to %s threw exception %s", uri, ex) + defer.returnValue(False) @defer.inlineCallbacks def push(self, service, event): diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2c6d4e2815..ef94215133 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -23,7 +23,6 @@ import synapse.util.stringutils as stringutils import logging - logger = logging.getLogger(__name__) @@ -58,6 +57,7 @@ class ApplicationServicesHandler(object): yield self.store.update_app_service(app_service) defer.returnValue(app_service) + @defer.inlineCallbacks def unregister(self, token): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) @@ -81,34 +81,45 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: - user_query_services = yield self._get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_USERS - ) - for user_service in user_query_services: - # this needs to block XXX: Need to feed response back to caller - is_known_user = yield self.appservice_api.query_user( - user_service, event.sender - ) - if is_known_user: - # the user exists now,so don't query more ASes. - break + yield self.query_user_exists(event) # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + @defer.inlineCallbacks + def query_user_exists(self, event): + """Check if an application services knows this event.sender exists. + + Args: + event: An event sent by the user to query + Returns: + True if this user exists. + """ + # TODO Would be nice for this to accept a user ID instead of an event. + user_query_services = yield self._get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_USERS + ) + for user_service in user_query_services: + is_known_user = yield self.appservice_api.query_user( + user_service, event.sender + ) + if is_known_user: + defer.returnValue(True) + defer.returnValue(False) @defer.inlineCallbacks def query_room_alias_exists(self, room_alias): """Check if an application service knows this room alias exists. Args: - room_alias(str): The room alias to query. + room_alias(RoomAlias): The room alias to query. Returns: namedtuple: with keys "room_id" and "servers" or None if no association can be found. """ + room_alias = room_alias.to_string() alias_query_services = yield self._get_services_for_event( event=None, restrict_to=ApplicationService.NS_ALIASES, diff --git a/synapse/notifier.py b/synapse/notifier.py index e3b6ead620..c7f75ab801 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -99,6 +99,12 @@ class Notifier(object): `extra_users` param. """ yield run_on_reactor() + + # poke any interested application service. + self.hs.get_handlers().appservice_handler.notify_interested_services( + event + ) + room_id = event.room_id room_source = self.event_sources.sources["room"] -- cgit 1.5.1 From 0613666d9c417005393636b935593c423f4417b9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 13:42:35 +0000 Subject: Serialize events before sending to ASes --- synapse/appservice/api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'synapse/appservice') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index d96caf7f58..9cce4e0973 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -16,6 +16,7 @@ from twisted.internet import defer from synapse.api.errors import CodeMessageException from synapse.http.client import SimpleHttpClient +from synapse.events.utils import serialize_event import logging import urllib @@ -30,6 +31,7 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) + self.clock = hs.get_clock() @defer.inlineCallbacks def query_user(self, service, user_id): @@ -72,6 +74,8 @@ class ApplicationServiceApi(SimpleHttpClient): @defer.inlineCallbacks def push_bulk(self, service, events): + events = self._serialize(events) + uri = service.url + ("/transactions/%s" % urllib.quote(str(0))) # TODO txn_ids response = None @@ -98,3 +102,9 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.push_bulk(service, [event]) defer.returnValue(response) + def _serialize(self, events): + time_now = self.clock.time_msec() + return [ + serialize_event(e, time_now, as_client_event=True) for e in events + ] + -- cgit 1.5.1 From c71456117dc70dfe0bfa15e3f655a3ac1dfc66ee Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 14:17:08 +0000 Subject: Fix user query checks. HS>AS pushing now works. --- synapse/appservice/__init__.py | 21 ++++++++++++-------- synapse/handlers/appservice.py | 44 +++++++++++++++++++++++++++++------------- 2 files changed, 44 insertions(+), 21 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index d9ca856c8b..46d46a5a48 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -75,27 +75,23 @@ class ApplicationService(object): def _matches_user(self, event): if (hasattr(event, "sender") and - self._matches_regex( - event.sender, ApplicationService.NS_USERS)): + self.is_interested_in_user(event.sender)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member and hasattr(event, "state_key") - and self._matches_regex( - event.state_key, ApplicationService.NS_USERS)): + and self.is_interested_in_user(event.state_key)): return True return False def _matches_room_id(self, event): if hasattr(event, "room_id"): - return self._matches_regex( - event.room_id, ApplicationService.NS_ROOMS - ) + return self.is_interested_in_room(event.room_id) return False def _matches_aliases(self, event, alias_list): for alias in alias_list: - if self._matches_regex(alias, ApplicationService.NS_ALIASES): + if self.is_interested_in_alias(alias): return True return False @@ -128,5 +124,14 @@ class ApplicationService(object): elif restrict_to == ApplicationService.NS_USERS: return self._matches_user(event) + def is_interested_in_user(self, user_id): + return self._matches_regex(user_id, ApplicationService.NS_USERS) + + def is_interested_in_alias(self, alias): + return self._matches_regex(alias, ApplicationService.NS_ALIASES) + + def is_interested_in_room(self, room_id): + return self._matches_regex(room_id, ApplicationService.NS_ROOMS) + def __str__(self): return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index ef94215133..8d0cdd528c 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,6 +15,7 @@ from twisted.internet import defer +from synapse.api.constants import EventTypes from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi @@ -78,32 +79,31 @@ class ApplicationServicesHandler(object): return # no services need notifying # Do we know this user exists? If not, poke the user query API for - # all services which match that user regex. - unknown_user = yield self._is_unknown_user(event.sender) - if unknown_user: - yield self.query_user_exists(event) + # all services which match that user regex. This needs to block as these + # user queries need to be made BEFORE pushing the event. + yield self._check_user_exists(event.sender) + if event.type == EventTypes.Member: + yield self._check_user_exists(event.state_key) # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) @defer.inlineCallbacks - def query_user_exists(self, event): - """Check if an application services knows this event.sender exists. + def query_user_exists(self, user_id): + """Check if any application service knows this user_id exists. Args: - event: An event sent by the user to query + user_id(str): The user to query if they exist on any AS. Returns: - True if this user exists. + True if this user exists on at least one application service. """ - # TODO Would be nice for this to accept a user ID instead of an event. - user_query_services = yield self._get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_USERS + user_query_services = yield self._get_services_for_user( + user_id=user_id ) for user_service in user_query_services: is_known_user = yield self.appservice_api.query_user( - user_service, event.sender + user_service, user_id ) if is_known_user: defer.returnValue(True) @@ -161,6 +161,16 @@ class ApplicationServicesHandler(object): ] defer.returnValue(interested_list) + @defer.inlineCallbacks + def _get_services_for_user(self, user_id): + services = yield self.store.get_app_services() + interested_list = [ + s for s in services if ( + s.is_interested_in_user(user_id) + ) + ] + defer.returnValue(interested_list) + @defer.inlineCallbacks def _is_unknown_user(self, user_id): user = UserID.from_string(user_id) @@ -173,5 +183,13 @@ class ApplicationServicesHandler(object): user_info = yield self.store.get_user_by_id(user_id) defer.returnValue(len(user_info) == 0) + @defer.inlineCallbacks + def _check_user_exists(self, user_id): + unknown_user = yield self._is_unknown_user(user_id) + if unknown_user: + exists = yield self.query_user_exists(user_id) + defer.returnValue(exists) + defer.returnValue(True) + def _generate_hs_token(self): return stringutils.random_string(24) -- cgit 1.5.1 From ac3183caaa66b750996d90c0ac9ed430f623909c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 12:03:37 +0000 Subject: Register a user account for the AS when the AS registers. Add 'sender' column to AS table. --- synapse/appservice/__init__.py | 3 ++- synapse/handlers/appservice.py | 8 +++++++- synapse/storage/appservice.py | 5 +++-- synapse/storage/schema/application_services.sql | 1 + synapse/storage/schema/delta/v14.sql | 1 + 5 files changed, 14 insertions(+), 4 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 46d46a5a48..fb9bfffe5d 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -35,10 +35,11 @@ class ApplicationService(object): NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None, hs_token=None, - txn_id=None): + sender=None, txn_id=None): self.token = token self.url = url self.hs_token = hs_token + self.sender = sender self.namespaces = self._check_namespaces(namespaces) self.txn_id = txn_id diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index fa810b9a98..5071a12eb1 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -52,8 +52,14 @@ class ApplicationServicesHandler(object): "Consult the home server admin.", errcode=Codes.FORBIDDEN ) - logger.info("Updating application service info...") + app_service.hs_token = self._generate_hs_token() + + # create a sender for this application service which is used when + # creating rooms, etc.. + account = yield self.hs.get_handlers().registration_handler.register() + app_service.sender = account[0] + yield self.store.update_app_service(app_service) defer.returnValue(app_service) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 3c8bf9ad0d..eef77e737e 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -130,8 +130,9 @@ class ApplicationServiceStore(SQLBaseStore): return False txn.execute( - "UPDATE application_services SET url=?, hs_token=? WHERE id=?", - (service.url, service.hs_token, as_id,) + "UPDATE application_services SET url=?, hs_token=?, sender=? " + "WHERE id=?", + (service.url, service.hs_token, service.sender, as_id,) ) # cleanup regex txn.execute( diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql index 03b5a10c8a..e491ad5aec 100644 --- a/synapse/storage/schema/application_services.sql +++ b/synapse/storage/schema/application_services.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS application_services( url TEXT, token TEXT, hs_token TEXT, + sender TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); diff --git a/synapse/storage/schema/delta/v14.sql b/synapse/storage/schema/delta/v14.sql index 03b5a10c8a..e491ad5aec 100644 --- a/synapse/storage/schema/delta/v14.sql +++ b/synapse/storage/schema/delta/v14.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS application_services( url TEXT, token TEXT, hs_token TEXT, + sender TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); -- cgit 1.5.1 From f7cac2f7b699fb3bb58c02fa1828aeec6c319f01 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 15:01:28 +0000 Subject: Fix bugs so lazy room joining works as intended. --- synapse/appservice/api.py | 6 +++--- synapse/handlers/appservice.py | 6 +++--- synapse/handlers/directory.py | 23 +++++++++++++++++------ 3 files changed, 23 insertions(+), 12 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 9cce4e0973..15ac1e27fc 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -41,7 +41,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": service.hs_token }) - if response: # just an empty json object + if response is not None: # just an empty json object defer.returnValue(True) except CodeMessageException as e: if e.code == 404: @@ -60,13 +60,13 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": service.hs_token }) - if response: # just an empty json object + if response is not None: # just an empty json object defer.returnValue(True) except CodeMessageException as e: + logger.warning("query_alias to %s received %s", uri, e.code) if e.code == 404: defer.returnValue(False) return - logger.warning("query_alias to %s received %s", uri, e.code) except Exception as ex: logger.warning("query_alias to %s threw exception %s", uri, ex) defer.returnValue(False) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 5071a12eb1..8591a77bf3 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -124,15 +124,15 @@ class ApplicationServicesHandler(object): namedtuple: with keys "room_id" and "servers" or None if no association can be found. """ - room_alias = room_alias.to_string() + room_alias_str = room_alias.to_string() alias_query_services = yield self._get_services_for_event( event=None, restrict_to=ApplicationService.NS_ALIASES, - alias_list=[room_alias] + alias_list=[room_alias_str] ) for alias_service in alias_query_services: is_known_alias = yield self.appservice_api.query_alias( - alias_service, room_alias + alias_service, room_alias_str ) if is_known_alias: # the alias exists now so don't query more ASes. diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 22c6391a15..87bc12c983 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -64,8 +64,11 @@ class DirectoryHandler(BaseHandler): # association creation for human users # TODO(erikj): Do user auth. - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: + can_create = yield self.can_modify_alias( + room_alias, + user_id=user_id + ) + if not can_create: raise SynapseError( 400, "This alias is reserved by an application service.", errcode=Codes.EXCLUSIVE @@ -91,8 +94,11 @@ class DirectoryHandler(BaseHandler): # TODO Check if server admin - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: + can_delete = yield self.can_modify_alias( + room_alias, + user_id=user_id + ) + if not can_delete: raise SynapseError( 400, "This alias is reserved by an application service.", errcode=Codes.EXCLUSIVE @@ -228,9 +234,14 @@ class DirectoryHandler(BaseHandler): defer.returnValue(result) @defer.inlineCallbacks - def is_alias_exclusive_to_appservices(self, alias): + def can_modify_alias(self, alias, user_id=None): services = yield self.store.get_app_services() interested_services = [ s for s in services if s.is_interested_in_alias(alias.to_string()) ] - defer.returnValue(len(interested_services) > 0) + for service in interested_services: + if user_id == service.sender: + # this user IS the app service + defer.returnValue(True) + return + defer.returnValue(len(interested_services) == 0) -- cgit 1.5.1 From c7783d6feec9b69c24f3303cbb51cce3e6b8ffb3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:36:08 +0000 Subject: Notify ASes for events sent by other users in a room which an AS user is a part of. --- synapse/appservice/__init__.py | 17 +++++++++++++---- synapse/handlers/appservice.py | 21 ++++++++++++++++----- synapse/rest/appservice/v1/__init__.py | 4 ++-- tests/appservice/test_appservice.py | 25 +++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 11 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index fb9bfffe5d..381b4cfc4a 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -74,7 +74,7 @@ class ApplicationService(object): return True return False - def _matches_user(self, event): + def _matches_user(self, event, member_list): if (hasattr(event, "sender") and self.is_interested_in_user(event.sender)): return True @@ -83,6 +83,10 @@ class ApplicationService(object): and hasattr(event, "state_key") and self.is_interested_in_user(event.state_key)): return True + # check joined member events + for member in member_list: + if self.is_interested_in_user(member.state_key): + return True return False def _matches_room_id(self, event): @@ -96,7 +100,8 @@ class ApplicationService(object): return True return False - def is_interested(self, event, restrict_to=None, aliases_for_event=None): + def is_interested(self, event, restrict_to=None, aliases_for_event=None, + member_list=None): """Check if this service is interested in this event. Args: @@ -104,18 +109,22 @@ class ApplicationService(object): restrict_to(str): The namespace to restrict regex tests to. aliases_for_event(list): A list of all the known room aliases for this event. + member_list(list): A list of all joined room members in this room. Returns: bool: True if this service would like to know about this event. """ if aliases_for_event is None: aliases_for_event = [] + if member_list is None: + member_list = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: # this is a programming error, so fail early and raise a general # exception raise Exception("Unexpected restrict_to value: %s". restrict_to) if not restrict_to: - return (self._matches_user(event) + return (self._matches_user(event, member_list) or self._matches_aliases(event, aliases_for_event) or self._matches_room_id(event)) elif restrict_to == ApplicationService.NS_ALIASES: @@ -123,7 +132,7 @@ class ApplicationService(object): elif restrict_to == ApplicationService.NS_ROOMS: return self._matches_room_id(event) elif restrict_to == ApplicationService.NS_USERS: - return self._matches_user(event) + return self._matches_user(event, member_list) def is_interested_in_user(self, user_id): return self._matches_regex(user_id, ApplicationService.NS_USERS) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8591a77bf3..2c488a46f6 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, Membership from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.types import UserID @@ -154,14 +154,25 @@ class ApplicationServicesHandler(object): list: A list of services interested in this event based on the service regex. """ - # We need to know the aliases associated with this event.room_id, if any - if not alias_list: - alias_list = yield self.store.get_aliases_for_room(event.room_id) + member_list = None + if hasattr(event, "room_id"): + # We need to know the aliases associated with this event.room_id, + # if any. + if not alias_list: + alias_list = yield self.store.get_aliases_for_room( + event.room_id + ) + # We need to know the members associated with this event.room_id, + # if any. + member_list = yield self.store.get_room_members( + room_id=event.room_id, + membership=Membership.JOIN + ) services = yield self.store.get_app_services() interested_list = [ s for s in services if ( - s.is_interested(event, restrict_to, alias_list) + s.is_interested(event, restrict_to, alias_list, member_list) ) ] defer.returnValue(interested_list) diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py index bf243b6180..a7877609ad 100644 --- a/synapse/rest/appservice/v1/__init__.py +++ b/synapse/rest/appservice/v1/__init__.py @@ -21,9 +21,9 @@ class AppServiceRestResource(JsonResource): """A resource for version 1 of the matrix application service API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod def register_servlets(appservice_resource, hs): - register.register_servlets(hs, appservice_resource) \ No newline at end of file + register.register_servlets(hs, appservice_resource) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index c0aaf12785..d12e4f2644 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -143,3 +143,28 @@ class ApplicationServiceTestCase(unittest.TestCase): restrict_to=ApplicationService.NS_USERS, aliases_for_event=["#xmpp_barfoo:matrix.org"] )) + + def test_member_list_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + join_list = [ + Mock( + type="m.room.member", room_id="!foo:bar", sender="@alice:here", + state_key="@alice:here" + ), + Mock( + type="m.room.member", room_id="!foo:bar", sender="@irc_fo:here", + state_key="@irc_fo:here" # AS user + ), + Mock( + type="m.room.member", room_id="!foo:bar", sender="@bob:here", + state_key="@bob:here" + ) + ] + + self.event.sender = "@xmpp_foobar:matrix.org" + self.assertTrue(self.service.is_interested( + event=self.event, + member_list=join_list + )) -- cgit 1.5.1 From fd40d992adfb8b63f6e925dad030c63498501408 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:41:33 +0000 Subject: PEP8-ify --- synapse/appservice/api.py | 2 -- synapse/handlers/directory.py | 3 --- synapse/rest/appservice/v1/register.py | 2 +- synapse/storage/appservice.py | 3 --- 4 files changed, 1 insertion(+), 9 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 15ac1e27fc..6192813c03 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -71,7 +71,6 @@ class ApplicationServiceApi(SimpleHttpClient): logger.warning("query_alias to %s threw exception %s", uri, ex) defer.returnValue(False) - @defer.inlineCallbacks def push_bulk(self, service, events): events = self._serialize(events) @@ -107,4 +106,3 @@ class ApplicationServiceApi(SimpleHttpClient): return [ serialize_event(e, time_now, as_client_event=True) for e in events ] - diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 87bc12c983..20ab9e269c 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -58,7 +58,6 @@ class DirectoryHandler(BaseHandler): servers ) - @defer.inlineCallbacks def create_association(self, user_id, room_alias, room_id, servers=None): # association creation for human users @@ -75,7 +74,6 @@ class DirectoryHandler(BaseHandler): ) yield self._create_association(room_alias, room_id, servers) - @defer.inlineCallbacks def create_appservice_association(self, service, room_alias, room_id, servers=None): @@ -127,7 +125,6 @@ class DirectoryHandler(BaseHandler): # if room_id: # yield self._update_room_alias_events(user_id, room_id) - @defer.inlineCallbacks def get_association(self, room_alias): room_id = None diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index d3d5aef220..3bd0c1220c 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -65,7 +65,7 @@ class RegisterRestServlet(AppServiceRestServlet): hs_token = app_service.hs_token defer.returnValue((200, { - "hs_token": hs_token + "hs_token": hs_token })) def _parse_namespace(self, target_ns, origin_ns, ns): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index ba31c68595..d941b1f387 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -97,7 +97,6 @@ class ApplicationServiceStore(SQLBaseStore): # allocate new ASes. It relies on the server admin inserting the AS # token into the database manually. - if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") @@ -186,7 +185,6 @@ class ApplicationServiceStore(SQLBaseStore): # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. - @defer.inlineCallbacks def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" @@ -244,4 +242,3 @@ class ApplicationServiceStore(SQLBaseStore): hs_token=service["hs_token"], sender=service["sender"] )) - -- cgit 1.5.1 From f51832442674a1f72c41ffce2279880109fc7ff0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 16:41:16 +0000 Subject: Minor tweaks based on PR feedback. --- synapse/appservice/api.py | 6 +++--- synapse/http/client.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 6192813c03..c2179f8d55 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -80,11 +80,11 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.put_json( - uri, - { + uri=uri, + json_body={ "events": events }, - { + args={ "access_token": service.hs_token }) if response: # just an empty json object diff --git a/synapse/http/client.py b/synapse/http/client.py index 575510637e..7b23116556 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -95,7 +95,8 @@ class SimpleHttpClient(object): Deferred: Succeeds when we get *any* 2xx HTTP response, with the HTTP body as JSON. Raises: - On a non-2xx HTTP response. + On a non-2xx HTTP response. The response body will be used as the + error message. """ if len(args): query_bytes = urllib.urlencode(args, True) -- cgit 1.5.1 From 16b90764adb8f2ab49b1853855d0fb739b79d245 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 27 Feb 2015 10:44:32 +0000 Subject: Convert expected format for AS regex to include exclusivity. Previously you just specified the regex as a string, now it expects a JSON object with a 'regex' key and an 'exclusive' boolean, as per spec. --- synapse/appservice/__init__.py | 26 ++++++++++++++++++------- synapse/rest/appservice/v1/register.py | 35 ++++++---------------------------- synapse/storage/appservice.py | 16 +++++++++++++--- 3 files changed, 38 insertions(+), 39 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 381b4cfc4a..b5e7ac16ba 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -46,19 +46,31 @@ class ApplicationService(object): def _check_namespaces(self, namespaces): # Sanity check that it is of the form: # { - # users: ["regex",...], - # aliases: ["regex",...], - # rooms: ["regex",...], + # users: [ {regex: "[A-z]+.*", exclusive: true}, ...], + # aliases: [ {regex: "[A-z]+.*", exclusive: true}, ...], + # rooms: [ {regex: "[A-z]+.*", exclusive: true}, ...], # } if not namespaces: return None for ns in ApplicationService.NS_LIST: + if ns not in namespaces: + namespaces[ns] = [] + continue + if type(namespaces[ns]) != list: - raise ValueError("Bad namespace value for '%s'", ns) - for regex in namespaces[ns]: - if not isinstance(regex, basestring): - raise ValueError("Expected string regex for ns '%s'", ns) + raise ValueError("Bad namespace value for '%s'" % ns) + for regex_obj in namespaces[ns]: + if not isinstance(regex_obj, dict): + raise ValueError("Expected dict regex for ns '%s'" % ns) + if not isinstance(regex_obj.get("exclusive"), bool): + raise ValueError( + "Expected bool for 'exclusive' in ns '%s'" % ns + ) + if not isinstance(regex_obj.get("regex"), basestring): + raise ValueError( + "Expected string for 'regex' in ns '%s'" % ns + ) return namespaces def _matches_regex(self, test_string, namespace_key): diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 3bd0c1220c..a4f6159773 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -48,18 +48,12 @@ class RegisterRestServlet(AppServiceRestServlet): 400, "Missed required keys: as_token(str) / url(str)." ) - namespaces = { - "users": [], - "rooms": [], - "aliases": [] - } - - if "namespaces" in params: - self._parse_namespace(namespaces, params["namespaces"], "users") - self._parse_namespace(namespaces, params["namespaces"], "rooms") - self._parse_namespace(namespaces, params["namespaces"], "aliases") - - app_service = ApplicationService(as_token, as_url, namespaces) + try: + app_service = ApplicationService( + as_token, as_url, params["namespaces"] + ) + except ValueError as e: + raise SynapseError(400, e.message) app_service = yield self.handler.register(app_service) hs_token = app_service.hs_token @@ -68,23 +62,6 @@ class RegisterRestServlet(AppServiceRestServlet): "hs_token": hs_token })) - def _parse_namespace(self, target_ns, origin_ns, ns): - if ns not in target_ns or ns not in origin_ns: - return # nothing to parse / map through to. - - possible_regex_list = origin_ns[ns] - if not type(possible_regex_list) == list: - raise SynapseError(400, "Namespace %s isn't an array." % ns) - - for regex in possible_regex_list: - if not isinstance(regex, basestring): - raise SynapseError( - 400, "Regex '%s' isn't a string in namespace %s" % - (regex, ns) - ) - - target_ns[ns] = origin_ns[ns] - class UnregisterRestServlet(AppServiceRestServlet): """Handles AS registration with the home server. diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index dc3666efd4..a3aa41e5fc 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import simplejson +from simplejson import JSONDecodeError from twisted.internet import defer from synapse.api.errors import StoreError @@ -23,12 +25,18 @@ from ._base import SQLBaseStore logger = logging.getLogger(__name__) +def log_failure(failure): + logger.error("Failed to detect application services: %s", failure.value) + logger.error(failure.getTraceback()) + + class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.services_cache = [] self.cache_defer = self._populate_cache() + self.cache_defer.addErrback(log_failure) @defer.inlineCallbacks def unregister_app_service(self, token): @@ -128,11 +136,11 @@ class ApplicationServiceStore(SQLBaseStore): ) for (ns_int, ns_str) in enumerate(ApplicationService.NS_LIST): if ns_str in service.namespaces: - for regex in service.namespaces[ns_str]: + for regex_obj in service.namespaces[ns_str]: txn.execute( "INSERT INTO application_services_regex(" "as_id, namespace, regex) values(?,?,?)", - (as_id, ns_int, regex) + (as_id, ns_int, simplejson.dumps(regex_obj)) ) return True @@ -215,10 +223,12 @@ class ApplicationServiceStore(SQLBaseStore): try: services[as_token]["namespaces"][ ApplicationService.NS_LIST[ns_int]].append( - res["regex"] + simplejson.loads(res["regex"]) ) except IndexError: logger.error("Bad namespace enum '%s'. %s", ns_int, res) + except JSONDecodeError: + logger.error("Bad regex object '%s'", res["regex"]) # TODO get last successful txn id f.e. service for service in services.values(): -- cgit 1.5.1 From 40c9896705208b1455192b496e3e8e3bc9e9c0a9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 27 Feb 2015 11:03:56 +0000 Subject: Add functions to return whether an AS has exclusively claimed a matching namespace. --- synapse/appservice/__init__.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) (limited to 'synapse/appservice') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index b5e7ac16ba..a268a6bcc4 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -73,7 +73,7 @@ class ApplicationService(object): ) return namespaces - def _matches_regex(self, test_string, namespace_key): + def _matches_regex(self, test_string, namespace_key, return_obj=False): if not isinstance(test_string, basestring): logger.error( "Expected a string to test regex against, but got %s", @@ -81,11 +81,19 @@ class ApplicationService(object): ) return False - for regex in self.namespaces[namespace_key]: - if re.match(regex, test_string): + for regex_obj in self.namespaces[namespace_key]: + if re.match(regex_obj["regex"], test_string): + if return_obj: + return regex_obj return True return False + def _is_exclusive(self, ns_key, test_string): + regex_obj = self._matches_regex(test_string, ns_key, return_obj=True) + if regex_obj: + return regex_obj["exclusive"] + return False + def _matches_user(self, event, member_list): if (hasattr(event, "sender") and self.is_interested_in_user(event.sender)): @@ -155,5 +163,14 @@ class ApplicationService(object): def is_interested_in_room(self, room_id): return self._matches_regex(room_id, ApplicationService.NS_ROOMS) + def is_exclusive_user(self, user_id): + return self._is_exclusive(ApplicationService.NS_USERS, user_id) + + def is_exclusive_alias(self, alias): + return self._is_exclusive(ApplicationService.NS_ALIASES, alias) + + def is_exclusive_room(self, room_id): + return self._is_exclusive(ApplicationService.NS_ROOMS, room_id) + def __str__(self): return "ApplicationService: %s" % (self.__dict__,) -- cgit 1.5.1