diff options
| -rw-r--r-- | synapse/api/urls.py | 1 | ||||
| -rwxr-xr-x | synapse/app/homeserver.py | 7 | ||||
| -rw-r--r-- | synapse/appservice/__init__.py | 130 | ||||
| -rw-r--r-- | synapse/appservice/api.py | 36 | ||||
| -rw-r--r-- | synapse/handlers/__init__.py | 2 | ||||
| -rw-r--r-- | synapse/handlers/appservice.py | 132 | ||||
| -rw-r--r-- | synapse/rest/appservice/__init__.py | 14 | ||||
| -rw-r--r-- | synapse/rest/appservice/v1/__init__.py | 29 | ||||
| -rw-r--r-- | synapse/rest/appservice/v1/base.py | 48 | ||||
| -rw-r--r-- | synapse/rest/appservice/v1/register.py | 121 | ||||
| -rw-r--r-- | synapse/server.py | 1 | ||||
| -rw-r--r-- | synapse/storage/__init__.py | 3 | ||||
| -rw-r--r-- | synapse/storage/appservice.py | 237 | ||||
| -rw-r--r-- | synapse/storage/schema/application_services.sql | 32 | ||||
| -rw-r--r-- | tests/appservice/__init__.py | 14 | ||||
| -rw-r--r-- | tests/appservice/test_appservice.py | 145 | ||||
| -rw-r--r-- | tests/handlers/test_appservice.py | 64 | ||||
| -rw-r--r-- | tests/storage/test_appservice.py | 105 |
18 files changed, 1120 insertions, 1 deletions
diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 693c0efda6..9485719332 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -22,3 +22,4 @@ WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_PREFIX = "/_matrix/key/v1" MEDIA_PREFIX = "/_matrix/media/v1" +APP_SERVICE_PREFIX = "/_matrix/appservice/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f20ccfb5b6..a9397de5b2 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,13 +26,14 @@ from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect +from synapse.rest.appservice.v1 import AppServiceRestResource from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, APP_SERVICE_PREFIX ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory @@ -69,6 +70,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_federation(self): return JsonResource() + def build_resource_for_app_services(self): + return AppServiceRestResource(self) + def build_resource_for_web_client(self): syweb_path = os.path.dirname(syweb.__file__) webclient_path = os.path.join(syweb_path, "webclient") @@ -114,6 +118,7 @@ class SynapseHomeServer(HomeServer): (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), (MEDIA_PREFIX, self.get_resource_for_media_repository()), + (APP_SERVICE_PREFIX, self.get_resource_for_app_services()), ] if web_client: logger.info("Adding the web client.") diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py new file mode 100644 index 0000000000..0c7f58574e --- /dev/null +++ b/synapse/appservice/__init__.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.api.constants import EventTypes + +import logging +import re + +logger = logging.getLogger(__name__) + + +class ApplicationService(object): + """Defines an application service. This definition is mostly what is + provided to the /register AS API. + + Provides methods to check if this service is "interested" in events. + """ + NS_USERS = "users" + NS_ALIASES = "aliases" + NS_ROOMS = "rooms" + # The ordering here is important as it is used to map database values (which + # are stored as ints representing the position in this list) to namespace + # values. + NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] + + def __init__(self, token, url=None, namespaces=None, txn_id=None): + self.token = token + self.url = url + self.namespaces = self._check_namespaces(namespaces) + self.txn_id = None + + def _check_namespaces(self, namespaces): + # Sanity check that it is of the form: + # { + # users: ["regex",...], + # aliases: ["regex",...], + # rooms: ["regex",...], + # } + if not namespaces: + return None + + for ns in ApplicationService.NS_LIST: + if type(namespaces[ns]) != list: + raise ValueError("Bad namespace value for '%s'", ns) + for regex in namespaces[ns]: + if not isinstance(regex, basestring): + raise ValueError("Expected string regex for ns '%s'", ns) + return namespaces + + def _matches_regex(self, test_string, namespace_key): + if not isinstance(test_string, basestring): + logger.warning( + "Expected a string to test regex against, but got %s", + test_string + ) + return False + + for regex in self.namespaces[namespace_key]: + if re.match(regex, test_string): + return True + return False + + def _matches_user(self, event): + if (hasattr(event, "sender") and + self._matches_regex( + event.sender, ApplicationService.NS_USERS)): + return True + # also check m.room.member state key + if (hasattr(event, "type") and event.type == EventTypes.Member + and hasattr(event, "state_key") + and self._matches_regex( + event.state_key, ApplicationService.NS_USERS)): + return True + return False + + def _matches_room_id(self, event): + if hasattr(event, "room_id"): + return self._matches_regex( + event.room_id, ApplicationService.NS_ROOMS + ) + return False + + def _matches_aliases(self, event, alias_list): + for alias in alias_list: + if self._matches_regex(alias, ApplicationService.NS_ALIASES): + return True + return False + + def is_interested(self, event, restrict_to=None, aliases_for_event=None): + """Check if this service is interested in this event. + + Args: + event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. + aliases_for_event(list): A list of all the known room aliases for + this event. + Returns: + bool: True if this service would like to know about this event. + """ + if aliases_for_event is None: + aliases_for_event = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so fail early and raise a general + # exception + raise Exception("Unexpected restrict_to value: %s". restrict_to) + + if not restrict_to: + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) + elif restrict_to == ApplicationService.NS_ALIASES: + return self._matches_aliases(event, aliases_for_event) + elif restrict_to == ApplicationService.NS_ROOMS: + return self._matches_room_id(event) + elif restrict_to == ApplicationService.NS_USERS: + return self._matches_user(event) + + def __str__(self): + return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py new file mode 100644 index 0000000000..158aded66e --- /dev/null +++ b/synapse/appservice/api.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class ApplicationServiceApi(object): + """This class manages HS -> AS communications, including querying and + pushing. + """ + + def __init__(self, hs): + self.hs_token = "_hs_token_" # TODO extract hs token + + def query_user(self, service, user_id): + pass + + def query_alias(self, service, alias): + pass + + def push_bulk(self, service, events): + pass + + def push(self, service, event): + pass + diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index a32eab9316..b31518bf62 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -26,6 +26,7 @@ from .presence import PresenceHandler from .directory import DirectoryHandler from .typing import TypingNotificationHandler from .admin import AdminHandler +from .appservice import ApplicationServicesHandler from .sync import SyncHandler @@ -52,4 +53,5 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) + self.appservice_handler = ApplicationServicesHandler(hs) self.sync_handler = SyncHandler(hs) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py new file mode 100644 index 0000000000..3188c60f3d --- /dev/null +++ b/synapse/handlers/appservice.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import BaseHandler +from synapse.api.errors import Codes, StoreError, SynapseError +from synapse.appservice import ApplicationService +from synapse.appservice.api import ApplicationServiceApi + +import logging + + +logger = logging.getLogger(__name__) + + +# NB: Purposefully not inheriting BaseHandler since that contains way too much +# setup code which this handler does not need or use. This makes testing a lot +# easier. +class ApplicationServicesHandler(object): + + def __init__(self, hs): + self.store = hs.get_datastore() + self.hs = hs + self.appservice_api = ApplicationServiceApi(hs) + + @defer.inlineCallbacks + def register(self, app_service): + logger.info("Register -> %s", app_service) + # check the token is recognised + try: + stored_service = yield self.store.get_app_service_by_token( + app_service.token + ) + if not stored_service: + raise StoreError(404, "Application service not found") + except StoreError: + raise SynapseError( + 403, "Unrecognised application services token. " + "Consult the home server admin.", + errcode=Codes.FORBIDDEN + ) + logger.info("Updating application service info...") + yield self.store.update_app_service(app_service) + + def unregister(self, token): + logger.info("Unregister as_token=%s", token) + yield self.store.unregister_app_service(token) + + def get_services_for_event(self, event, restrict_to=""): + """Retrieve a list of application services interested in this event. + + Args: + event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. + Returns: + list<ApplicationService>: A list of services interested in this + event based on the service regex. + """ + # We need to know the aliases associated with this event.room_id, if any + alias_list = [] # TODO + + interested_list = [ + s for s in self.store.get_app_services() if ( + s.is_interested(event, restrict_to, alias_list) + ) + ] + return interested_list + + def notify_interested_services(self, event): + """Notifies (pushes) all application services interested in this event. + + Pushing is done asynchronously, so this method won't block for any + prolonged length of time. + + Args: + event(Event): The event to push out to interested services. + """ + # Gather interested services + services = self.get_services_for_event(event) + if len(services) == 0: + return # no services need notifying + + # Do we know this user exists? If not, poke the user query API for + # all services which match that user regex. + unknown_user = False # TODO check + if unknown_user: + user_query_services = self.get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_USERS + ) + for user_service in user_query_services: + # this needs to block XXX: Need to feed response back to caller + is_known_user = self.appservice_api.query_user( + user_service, event + ) + if is_known_user: + # the user exists now,so don't query more ASes. + break + + # Do we know this room alias exists? If not, poke the room alias query + # API for all services which match that room alias regex. + unknown_room_alias = False # TODO check + if unknown_room_alias: + alias_query_services = self.get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_ALIASES + ) + for alias_service in alias_query_services: + # this needs to block XXX: Need to feed response back to caller + is_known_alias = self.appservice_api.query_alias( + alias_service, event + ) + if is_known_alias: + # the alias exists now so don't query more ASes. + break + + # Fork off pushes to these services - XXX First cut, best effort + for service in services: + self.appservice_api.push(service, event) diff --git a/synapse/rest/appservice/__init__.py b/synapse/rest/appservice/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/synapse/rest/appservice/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py new file mode 100644 index 0000000000..bf243b6180 --- /dev/null +++ b/synapse/rest/appservice/v1/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from . import register + +from synapse.http.server import JsonResource + + +class AppServiceRestResource(JsonResource): + """A resource for version 1 of the matrix application service API.""" + + def __init__(self, hs): + JsonResource.__init__(self) + self.register_servlets(self, hs) + + @staticmethod + def register_servlets(appservice_resource, hs): + register.register_servlets(hs, appservice_resource) \ No newline at end of file diff --git a/synapse/rest/appservice/v1/base.py b/synapse/rest/appservice/v1/base.py new file mode 100644 index 0000000000..65d5bcf9be --- /dev/null +++ b/synapse/rest/appservice/v1/base.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.http.servlet import RestServlet +from synapse.api.urls import APP_SERVICE_PREFIX +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def as_path_pattern(path_regex): + """Creates a regex compiled appservice path with the correct path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + APP_SERVICE_PREFIX + path_regex) + + +class AppServiceRestServlet(RestServlet): + """A base Synapse REST Servlet for the application services version 1 API. + """ + + def __init__(self, hs): + self.hs = hs + self.handler = hs.get_handlers().appservice_handler diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py new file mode 100644 index 0000000000..e374d538e7 --- /dev/null +++ b/synapse/rest/appservice/v1/register.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains REST servlets to do with registration: /register""" +from twisted.internet import defer + +from base import AppServiceRestServlet, as_path_pattern +from synapse.api.errors import CodeMessageException, SynapseError +from synapse.storage.appservice import ApplicationService + +import json +import logging + +logger = logging.getLogger(__name__) + + +class RegisterRestServlet(AppServiceRestServlet): + """Handles AS registration with the home server. + """ + + PATTERN = as_path_pattern("/register$") + + @defer.inlineCallbacks + def on_POST(self, request): + params = _parse_json(request) + + # sanity check required params + try: + as_token = params["as_token"] + as_url = params["url"] + if (not isinstance(as_token, basestring) or + not isinstance(as_url, basestring)): + raise ValueError + except (KeyError, ValueError): + raise SynapseError( + 400, "Missed required keys: as_token(str) / url(str)." + ) + + namespaces = { + "users": [], + "rooms": [], + "aliases": [] + } + + if "namespaces" in params: + self._parse_namespace(namespaces, params["namespaces"], "users") + self._parse_namespace(namespaces, params["namespaces"], "rooms") + self._parse_namespace(namespaces, params["namespaces"], "aliases") + + app_service = ApplicationService(as_token, as_url, namespaces) + + yield self.handler.register(app_service) + hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? + + defer.returnValue((200, { + "hs_token": hs_token + })) + + def _parse_namespace(self, target_ns, origin_ns, ns): + if ns not in target_ns or ns not in origin_ns: + return # nothing to parse / map through to. + + possible_regex_list = origin_ns[ns] + if not type(possible_regex_list) == list: + raise SynapseError(400, "Namespace %s isn't an array." % ns) + + for regex in possible_regex_list: + if not isinstance(regex, basestring): + raise SynapseError( + 400, "Regex '%s' isn't a string in namespace %s" % + (regex, ns) + ) + + target_ns[ns] = origin_ns[ns] + + +class UnregisterRestServlet(AppServiceRestServlet): + """Handles AS registration with the home server. + """ + + PATTERN = as_path_pattern("/unregister$") + + def on_POST(self, request): + params = _parse_json(request) + try: + as_token = params["as_token"] + if not isinstance(as_token, basestring): + raise ValueError + except (KeyError, ValueError): + raise SynapseError(400, "Missing required key: as_token(str)") + + yield self.handler.unregister(as_token) + + raise CodeMessageException(500, "Not implemented") + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.") + return content + except ValueError: + raise SynapseError(400, "Content not JSON.") + + +def register_servlets(hs, http_server): + RegisterRestServlet(hs).register(http_server) + UnregisterRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index 23bdad0c7c..ba2b2593f1 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -77,6 +77,7 @@ class BaseHomeServer(object): 'resource_for_content_repo', 'resource_for_server_key', 'resource_for_media_repository', + 'resource_for_app_services', 'event_sources', 'ratelimiter', 'keyring', diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7c54b1b9d3..6ff0093136 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.api.constants import EventTypes +from .appservice import ApplicationServiceStore from .directory import DirectoryStore from .feedback import FeedbackStore from .presence import PresenceStore @@ -66,6 +67,7 @@ SCHEMAS = [ "event_signatures", "pusher", "media_repository", + "application_services", "filtering", "rejections", ] @@ -87,6 +89,7 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, + ApplicationServiceStore, EventFederationStore, MediaRepositoryStore, RejectionsStore, diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py new file mode 100644 index 0000000000..b64416de28 --- /dev/null +++ b/synapse/storage/appservice.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from twisted.internet import defer + +from synapse.api.errors import StoreError +from synapse.appservice import ApplicationService +from ._base import SQLBaseStore + + +logger = logging.getLogger(__name__) + + +class ApplicationServiceCache(object): + """Caches ApplicationServices and provides utility functions on top. + + This class is designed to be invoked on incoming events in order to avoid + hammering the database every time to extract a list of application service + regexes. + """ + + def __init__(self): + self.services = [] + + +class ApplicationServiceStore(SQLBaseStore): + + def __init__(self, hs): + super(ApplicationServiceStore, self).__init__(hs) + self.cache = ApplicationServiceCache() + self.cache_defer = self._populate_cache() + + @defer.inlineCallbacks + def unregister_app_service(self, token): + """Unregisters this service. + + This removes all AS specific regex and the base URL. The token is the + only thing preserved for future registration attempts. + """ + yield self.cache_defer # make sure the cache is ready + yield self.runInteraction( + "unregister_app_service", + self._unregister_app_service_txn, + token, + ) + # update cache TODO: Should this be in the txn? + for service in self.cache.services: + if service.token == token: + service.url = None + service.namespaces = None + + def _unregister_app_service_txn(self, txn, token): + # kill the url to prevent pushes + txn.execute( + "UPDATE application_services SET url=NULL WHERE token=?", + (token,) + ) + + # cleanup regex + as_id = self._get_as_id_txn(txn, token) + if not as_id: + logger.warning( + "unregister_app_service_txn: Failed to find as_id for token=", + token + ) + return False + + txn.execute( + "DELETE FROM application_services_regex WHERE as_id=?", + (as_id,) + ) + return True + + @defer.inlineCallbacks + def update_app_service(self, service): + """Update an application service, clobbering what was previously there. + + Args: + service(ApplicationService): The updated service. + """ + yield self.cache_defer # make sure the cache is ready + + # NB: There is no "insert" since we provide no public-facing API to + # allocate new ASes. It relies on the server admin inserting the AS + # token into the database manually. + + + if not service.token or not service.url: + raise StoreError(400, "Token and url must be specified.") + + yield self.runInteraction( + "update_app_service", + self._update_app_service_txn, + service + ) + + # update cache TODO: Should this be in the txn? + for (index, cache_service) in enumerate(self.cache.services): + if service.token == cache_service.token: + self.cache.services[index] = service + logger.info("Updated: %s", service) + return + # new entry + self.cache.services.append(service) + logger.info("Updated(new): %s", service) + + def _update_app_service_txn(self, txn, service): + as_id = self._get_as_id_txn(txn, service.token) + if not as_id: + logger.warning( + "update_app_service_txn: Failed to find as_id for token=", + service.token + ) + return False + + txn.execute( + "UPDATE application_services SET url=? WHERE id=?", + (service.url, as_id,) + ) + # cleanup regex + txn.execute( + "DELETE FROM application_services_regex WHERE as_id=?", + (as_id,) + ) + for (ns_int, ns_str) in enumerate(ApplicationService.NS_LIST): + if ns_str in service.namespaces: + for regex in service.namespaces[ns_str]: + txn.execute( + "INSERT INTO application_services_regex(" + "as_id, namespace, regex) values(?,?,?)", + (as_id, ns_int, regex) + ) + return True + + def _get_as_id_txn(self, txn, token): + cursor = txn.execute( + "SELECT id FROM application_services WHERE token=?", + (token,) + ) + res = cursor.fetchone() + if res: + return res[0] + + @defer.inlineCallbacks + def get_app_services(self): + yield self.cache_defer # make sure the cache is ready + defer.returnValue(self.cache.services) + + @defer.inlineCallbacks + def get_app_service_by_token(self, token, from_cache=True): + """Get the application service with the given token. + + Args: + token (str): The application service token. + from_cache (bool): True to get this service from the cache, False to + check the database. + Raises: + StoreError if there was a problem retrieving this service. + """ + yield self.cache_defer # make sure the cache is ready + + if from_cache: + for service in self.cache.services: + if service.token == token: + defer.returnValue(service) + return + defer.returnValue(None) + + # TODO: The from_cache=False impl + # TODO: This should be JOINed with the application_services_regex table. + + + @defer.inlineCallbacks + def _populate_cache(self): + """Populates the ApplicationServiceCache from the database.""" + sql = ("SELECT * FROM application_services LEFT JOIN " + "application_services_regex ON application_services.id = " + "application_services_regex.as_id") + # SQL results in the form: + # [ + # { + # 'regex': "something", + # 'url': "something", + # 'namespace': enum, + # 'as_id': 0, + # 'token': "something", + # 'id': 0 + # } + # ] + services = {} + results = yield self._execute_and_decode(sql) + for res in results: + as_token = res["token"] + if as_token not in services: + # add the service + services[as_token] = { + "url": res["url"], + "token": as_token, + "namespaces": { + ApplicationService.NS_USERS: [], + ApplicationService.NS_ALIASES: [], + ApplicationService.NS_ROOMS: [] + } + } + # add the namespace regex if one exists + ns_int = res["namespace"] + if ns_int is None: + continue + try: + services[as_token]["namespaces"][ + ApplicationService.NS_LIST[ns_int]].append( + res["regex"] + ) + except IndexError: + logger.error("Bad namespace enum '%s'. %s", ns_int, res) + + # TODO get last successful txn id f.e. service + for service in services.values(): + logger.info("Found application service: %s", service) + self.cache.services.append(ApplicationService( + service["token"], + service["url"], + service["namespaces"] + )) + diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql new file mode 100644 index 0000000000..6d245fc807 --- /dev/null +++ b/synapse/storage/schema/application_services.sql @@ -0,0 +1,32 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT, + token TEXT, + UNIQUE(token) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id INTEGER PRIMARY KEY AUTOINCREMENT, + as_id INTEGER NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex TEXT, + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + + + diff --git a/tests/appservice/__init__.py b/tests/appservice/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/tests/appservice/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py new file mode 100644 index 0000000000..c0aaf12785 --- /dev/null +++ b/tests/appservice/test_appservice.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.appservice import ApplicationService + +from mock import Mock, PropertyMock +from tests import unittest + + +class ApplicationServiceTestCase(unittest.TestCase): + + def setUp(self): + self.service = ApplicationService( + url="some_url", + token="some_token", + namespaces={ + ApplicationService.NS_USERS: [], + ApplicationService.NS_ROOMS: [], + ApplicationService.NS_ALIASES: [] + } + ) + self.event = Mock( + type="m.something", room_id="!foo:bar", sender="@someone:somewhere" + ) + + def test_regex_user_id_prefix_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_user_id_prefix_no_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.assertFalse(self.service.is_interested(self.event)) + + def test_regex_room_member_is_checked(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.event.type = "m.room.member" + self.event.state_key = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_room_id_match(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + "!some_prefix.*some_suffix:matrix.org" + ) + self.event.room_id = "!some_prefixs0m3th1nGsome_suffix:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_room_id_no_match(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + "!some_prefix.*some_suffix:matrix.org" + ) + self.event.room_id = "!XqBunHwQIXUiqCaoxq:matrix.org" + self.assertFalse(self.service.is_interested(self.event)) + + def test_regex_alias_match(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#irc_.*:matrix.org" + ) + self.assertTrue(self.service.is_interested( + self.event, + aliases_for_event=["#irc_foobar:matrix.org", "#athing:matrix.org"] + )) + + def test_regex_alias_no_match(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#irc_.*:matrix.org" + ) + self.assertFalse(self.service.is_interested( + self.event, + aliases_for_event=["#xmpp_foobar:matrix.org", "#athing:matrix.org"] + )) + + def test_regex_multiple_matches(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#irc_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested( + self.event, + aliases_for_event=["#irc_barfoo:matrix.org"] + )) + + def test_restrict_to_rooms(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + "!flibble_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.event.room_id = "!wibblewoo:matrix.org" + self.assertFalse(self.service.is_interested( + self.event, + restrict_to=ApplicationService.NS_ROOMS + )) + + def test_restrict_to_aliases(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#xmpp_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertFalse(self.service.is_interested( + self.event, + restrict_to=ApplicationService.NS_ALIASES, + aliases_for_event=["#irc_barfoo:matrix.org"] + )) + + def test_restrict_to_senders(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#xmpp_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@xmpp_foobar:matrix.org" + self.assertFalse(self.service.is_interested( + self.event, + restrict_to=ApplicationService.NS_USERS, + aliases_for_event=["#xmpp_barfoo:matrix.org"] + )) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py new file mode 100644 index 0000000000..1daa314f20 --- /dev/null +++ b/tests/handlers/test_appservice.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer +from .. import unittest + +from synapse.handlers.appservice import ApplicationServicesHandler + +from mock import Mock + + +class AppServiceHandlerTestCase(unittest.TestCase): + """ Tests the ApplicationServicesHandler. """ + + def setUp(self): + self.mock_store = Mock() + self.mock_as_api = Mock() + hs = Mock() + hs.get_datastore = Mock(return_value=self.mock_store) + self.handler = ApplicationServicesHandler(hs) # thing being tested + + # FIXME Would be nice to DI this rather than monkey patch:( + if not hasattr(self.handler, "appservice_api"): + # someone probably updated the handler but not the tests. Fail fast. + raise Exception("Test expected handler.appservice_api to exist.") + self.handler.appservice_api = self.mock_as_api + + def test_notify_interested_services(self): + interested_service = self._mkservice(is_interested=True) + services = [ + self._mkservice(is_interested=False), + interested_service, + self._mkservice(is_interested=False) + ] + + self.mock_store.get_app_services = Mock(return_value=services) + + event = Mock( + sender="@someone:anywhere", + type="m.room.message", + room_id="!foo:bar" + ) + self.mock_as_api.push = Mock() + self.handler.notify_interested_services(event) + self.mock_as_api.push.assert_called_once_with(interested_service, event) + + def _mkservice(self, is_interested): + service = Mock() + service.is_interested = Mock(return_value=is_interested) + service.token = "mock_service_token" + service.url = "mock_service_url" + return service diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py new file mode 100644 index 0000000000..56fdda377c --- /dev/null +++ b/tests/storage/test_appservice.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from tests import unittest +from twisted.internet import defer + +from synapse.appservice import ApplicationService +from synapse.server import HomeServer +from synapse.storage.appservice import ApplicationServiceStore + +from tests.utils import SQLiteMemoryDbPool, MockClock + + +class ApplicationServiceStoreTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer("test", db_pool=db_pool, clock=MockClock()) + self.as_token = "token1" + db_pool.runQuery( + "INSERT INTO application_services(token) VALUES(?)", + (self.as_token,) + ) + db_pool.runQuery( + "INSERT INTO application_services(token) VALUES(?)", ("token2",) + ) + db_pool.runQuery( + "INSERT INTO application_services(token) VALUES(?)", ("token3",) + ) + # must be done after inserts + self.store = ApplicationServiceStore(hs) + + @defer.inlineCallbacks + def test_update_and_retrieval_of_service(self): + url = "https://matrix.org/appservices/foobar" + user_regex = ["@foobar_.*:matrix.org"] + alias_regex = ["#foobar_.*:matrix.org"] + room_regex = [] + service = ApplicationService(url=url, token=self.as_token, namespaces={ + ApplicationService.NS_USERS: user_regex, + ApplicationService.NS_ALIASES: alias_regex, + ApplicationService.NS_ROOMS: room_regex + }) + yield self.store.update_app_service(service) + + stored_service = yield self.store.get_app_service_by_token( + self.as_token + ) + self.assertEquals(stored_service.token, self.as_token) + self.assertEquals(stored_service.url, url) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ALIASES], + alias_regex + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ROOMS], + room_regex + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_USERS], + user_regex + ) + + @defer.inlineCallbacks + def test_retrieve_unknown_service_token(self): + service = yield self.store.get_app_service_by_token("invalid_token") + self.assertEquals(service, None) + + @defer.inlineCallbacks + def test_retrieval_of_service(self): + stored_service = yield self.store.get_app_service_by_token( + self.as_token + ) + self.assertEquals(stored_service.token, self.as_token) + self.assertEquals(stored_service.url, None) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ALIASES], + [] + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ROOMS], + [] + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_USERS], + [] + ) + + @defer.inlineCallbacks + def test_retrieval_of_all_services(self): + services = yield self.store.get_app_services() + self.assertEquals(len(services), 3) |