From 74c38797601f6d7d1a02d21fc54ceb1a54629c64 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 19 Nov 2014 18:20:59 +0000 Subject: Start creating a module to do generic notifications (just prints them to stdout currently!) --- synapse/api/errors.py | 1 + synapse/app/homeserver.py | 2 + synapse/push/__init__.py | 76 ++++++++++++++++++++++++++++ synapse/push/httppusher.py | 40 +++++++++++++++ synapse/push/pusherpool.py | 94 +++++++++++++++++++++++++++++++++++ synapse/rest/__init__.py | 3 +- synapse/rest/pusher.py | 71 +++++++++++++++++++++++++++ synapse/server.py | 5 ++ synapse/storage/__init__.py | 6 ++- synapse/storage/pusher.py | 98 +++++++++++++++++++++++++++++++++++++ synapse/storage/schema/delta/v7.sql | 28 +++++++++++ synapse/storage/schema/pusher.sql | 28 +++++++++++ 12 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 synapse/push/__init__.py create mode 100644 synapse/push/httppusher.py create mode 100644 synapse/push/pusherpool.py create mode 100644 synapse/rest/pusher.py create mode 100644 synapse/storage/pusher.py create mode 100644 synapse/storage/schema/delta/v7.sql create mode 100644 synapse/storage/schema/pusher.sql (limited to 'synapse') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 33d15072af..97750ca2b0 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -32,6 +32,7 @@ class Codes(object): LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" CAPTCHA_INVALID = "M_CAPTCHA_INVALID" + MISSING_PARAM = "M_MISSING_PARAM" class CodeMessageException(Exception): diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 85284a4919..de16d8a2e1 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -242,6 +242,8 @@ def setup(): bind_port = None hs.start_listening(bind_port, config.unsecure_port) + hs.get_pusherpool().start() + if config.daemonize: print config.pid_file daemon = Daemonize( diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py new file mode 100644 index 0000000000..df0b91a8e9 --- /dev/null +++ b/synapse/push/__init__.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.streams.config import PaginationConfig +from synapse.types import StreamToken + +import synapse.util.async + +import logging + +logger = logging.getLogger(__name__) + +class Pusher(object): + INITIAL_BACKOFF = 1000 + MAX_BACKOFF = 10 * 60 * 1000 + + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + self.hs = _hs + self.evStreamHandler = self.hs.get_handlers().event_stream_handler + self.store = self.hs.get_datastore() + self.user_name = user_name + self.app = app + self.app_display_name = app_display_name + self.device_display_name = device_display_name + self.pushkey = pushkey + self.data = data + self.last_token = last_token + self.backoff_delay = Pusher.INITIAL_BACKOFF + + @defer.inlineCallbacks + def start(self): + if not self.last_token: + # First-time setup: get a token to start from (we can't just start from no token, ie. 'now' + # because we need the result to be reproduceable in case we fail to dispatch the push) + config = PaginationConfig(from_token=None, limit='1') + chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=0) + self.last_token = chunk['end'] + self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + logger.info("Pusher %s for user %s starting from token %s", + self.pushkey, self.user_name, self.last_token) + + while True: + from_tok = StreamToken.from_string(self.last_token) + config = PaginationConfig(from_token=from_tok, limit='1') + chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=100*365*24*60*60*1000) + + if (self.dispatchPush(chunk['chunk'][0])): + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + else: + logger.warn("Failed to dispatch push for user %s. Trying again in %dms", + self.user_name, self.backoff_delay) + yield synapse.util.async.sleep(self.backoff_delay / 1000.0) + self.backoff_delay *=2 + if self.backoff_delay > Pusher.MAX_BACKOFF: + self.backoff_delay = Pusher.MAX_BACKOFF + + +class PusherConfigException(Exception): + def __init__(self, msg): + super(PusherConfigException, self).__init__(msg) \ No newline at end of file diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py new file mode 100644 index 0000000000..988c4e32f5 --- /dev/null +++ b/synapse/push/httppusher.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.push import Pusher, PusherConfigException +from synapse.http.client import + +import logging + +logger = logging.getLogger(__name__) + +class HttpPusher(Pusher): + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + super(HttpPusher, self).__init__(_hs, + user_name, + app, + app_display_name, + device_display_name, + pushkey, + data, + last_token) + if 'url' not in data: + raise PusherConfigException("'url' required in data for HTTP pusher") + self.url = data['url'] + + def dispatchPush(self, event): + print event + return True + diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py new file mode 100644 index 0000000000..436040f123 --- /dev/null +++ b/synapse/push/pusherpool.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from httppusher import HttpPusher +from synapse.push import PusherConfigException + +import logging +import json + +logger = logging.getLogger(__name__) + +class PusherPool: + def __init__(self, _hs): + self.hs = _hs + self.store = self.hs.get_datastore() + self.pushers = [] + self.last_pusher_started = -1 + + def start(self): + self._pushers_added() + + def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + # we try to create the pusher just to validate the config: it will then get pulled out of the database, + # recreated, added and started: this means we have only one code path adding pushers. + self._create_pusher({ + "user_name": user_name, + "kind": kind, + "app": app, + "app_display_name": app_display_name, + "device_display_name": device_display_name, + "pushkey": pushkey, + "data": data, + "last_token": None + }) + self._add_pusher_to_store(user_name, kind, app, app_display_name, device_display_name, pushkey, data) + + @defer.inlineCallbacks + def _add_pusher_to_store(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + yield self.store.add_pusher(user_name=user_name, + kind=kind, + app=app, + app_display_name=app_display_name, + device_display_name=device_display_name, + pushkey=pushkey, + data=json.dumps(data)) + self._pushers_added() + + def _create_pusher(self, pusherdict): + if pusherdict['kind'] == 'http': + return HttpPusher(self.hs, + user_name=pusherdict['user_name'], + app=pusherdict['app'], + app_display_name=pusherdict['app_display_name'], + device_display_name=pusherdict['device_display_name'], + pushkey=pusherdict['pushkey'], + data=pusherdict['data'], + last_token=pusherdict['last_token'] + ) + else: + raise PusherConfigException("Unknown pusher type '%s' for user %s" % + (pusherdict['kind'], pusherdict['user_name'])) + + @defer.inlineCallbacks + def _pushers_added(self): + pushers = yield self.store.get_all_pushers_after_id(self.last_pusher_started) + for p in pushers: + p['data'] = json.loads(p['data']) + if (len(pushers)): + self.last_pusher_started = pushers[-1]['id'] + + self._start_pushers(pushers) + + def _start_pushers(self, pushers): + logger.info("Starting %d pushers", (len(pushers))) + for pusherdict in pushers: + p = self._create_pusher(pusherdict) + if p: + self.pushers.append(p) + p.start() \ No newline at end of file diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index e391e5678d..c38cf27690 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -16,7 +16,7 @@ from . import ( room, events, register, login, profile, presence, initial_sync, directory, - voip, admin, + voip, admin, pusher, ) @@ -45,3 +45,4 @@ class RestServletFactory(object): directory.register_servlets(hs, client_resource) voip.register_servlets(hs, client_resource) admin.register_servlets(hs, client_resource) + pusher.register_servlets(hs, client_resource) diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py new file mode 100644 index 0000000000..85d0d1c8cd --- /dev/null +++ b/synapse/rest/pusher.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import SynapseError, Codes +from synapse.push import PusherConfigException +from base import RestServlet, client_path_pattern + +import json + + +class PusherRestServlet(RestServlet): + PATTERN = client_path_pattern("/pushers/(?P[\w]*)$") + + @defer.inlineCallbacks + def on_PUT(self, request, pushkey): + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + reqd = ['kind', 'app', 'app_display_name', 'device_display_name', 'data'] + missing = [] + for i in reqd: + if i not in content: + missing.append(i) + if len(missing): + raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) + + pusher_pool = self.hs.get_pusherpool() + try: + pusher_pool.add_pusher(user_name=user.to_string(), + kind=content['kind'], + app=content['app'], + app_display_name=content['app_display_name'], + device_display_name=content['device_display_name'], + pushkey=pushkey, + data=content['data']) + except PusherConfigException as pce: + raise SynapseError(400, "Config Error: "+pce.message, errcode=Codes.MISSING_PARAM) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request): + return (200, {}) + +# XXX: C+ped from rest/room.py - surely this should be common? +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + +def register_servlets(hs, http_server): + PusherRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index da0a44433a..cfbe7d5e38 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -36,6 +36,7 @@ from synapse.util.lockutils import LockManager from synapse.streams.events import EventSources from synapse.api.ratelimiting import Ratelimiter from synapse.crypto.keyring import Keyring +from synapse.push.pusherpool import PusherPool class BaseHomeServer(object): @@ -82,6 +83,7 @@ class BaseHomeServer(object): 'ratelimiter', 'keyring', 'event_validator', + 'pusherpool' ] def __init__(self, hostname, **kwargs): @@ -228,6 +230,9 @@ class HomeServer(BaseHomeServer): def build_event_validator(self): return EventValidator(self) + def build_pusherpool(self): + return PusherPool(self) + def register_servlets(self): """ Register all servlets associated with this HomeServer. """ diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c36d938d96..5957f938a4 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -33,6 +33,7 @@ from .stream import StreamStore from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore +from .pusher import PusherStore from .state import StateStore from .signatures import SignatureStore @@ -62,12 +63,13 @@ SCHEMAS = [ "state", "event_edges", "event_signatures", + "pusher" ] # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 6 +SCHEMA_VERSION = 7 class _RollbackButIsFineException(Exception): @@ -81,7 +83,7 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, - EventFederationStore, ): + EventFederationStore, PusherStore, ): def __init__(self, hs): super(DataStore, self).__init__(hs) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py new file mode 100644 index 0000000000..047a5f42d9 --- /dev/null +++ b/synapse/storage/pusher.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from ._base import SQLBaseStore, Table +from twisted.internet import defer + +from sqlite3 import IntegrityError +from synapse.api.errors import StoreError + +import logging + +logger = logging.getLogger(__name__) + +class PusherStore(SQLBaseStore): + @defer.inlineCallbacks + def get_all_pushers_after_id(self, min_id): + sql = ( + "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, last_token " + "FROM pushers " + "WHERE id > ?" + ) + + rows = yield self._execute(None, sql, min_id) + + ret = [ + { + "id": r[0], + "user_name": r[1], + "kind": r[2], + "app": r[3], + "app_display_name": r[4], + "device_display_name": r[5], + "pushkey": r[6], + "data": r[7], + "last_token": r[8] + + } + for r in rows + ] + + defer.returnValue(ret) + + @defer.inlineCallbacks + def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + try: + yield self._simple_insert(PushersTable.table_name, dict( + user_name=user_name, + kind=kind, + app=app, + app_display_name=app_display_name, + device_display_name=device_display_name, + pushkey=pushkey, + data=data + )) + except IntegrityError: + raise StoreError(409, "Pushkey in use.") + except Exception as e: + logger.error("create_pusher with failed: %s", e) + raise StoreError(500, "Problem creating pusher.") + + @defer.inlineCallbacks + def update_pusher_last_token(self, user_name, pushkey, last_token): + yield self._simple_update_one(PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token} + ) + + +class PushersTable(Table): + table_name = "pushers" + + fields = [ + "id", + "user_name", + "kind", + "app" + "app_display_name", + "device_display_name", + "pushkey", + "data", + "last_token" + ] + + EntryType = collections.namedtuple("PusherEntry", fields) \ No newline at end of file diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql new file mode 100644 index 0000000000..7f6852485d --- /dev/null +++ b/synapse/storage/schema/delta/v7.sql @@ -0,0 +1,28 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + kind varchar(8) NOT NULL, + app varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + data text, + last_token TEXT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (user_name, pushkey) +); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql new file mode 100644 index 0000000000..7f6852485d --- /dev/null +++ b/synapse/storage/schema/pusher.sql @@ -0,0 +1,28 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + kind varchar(8) NOT NULL, + app varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + data text, + last_token TEXT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (user_name, pushkey) +); -- cgit 1.4.1 From 051b18581109022d79d9f270d5f5e565688d89d0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 19 Nov 2014 18:37:00 +0000 Subject: remove random half-line --- synapse/push/httppusher.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 988c4e32f5..f3c3ca8191 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -14,7 +14,6 @@ # limitations under the License. from synapse.push import Pusher, PusherConfigException -from synapse.http.client import import logging -- cgit 1.4.1 From eb6aedf92c0fe467fd4724623262907ad78573bb Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Nov 2014 12:21:00 +0000 Subject: More work on pushers. Attempt to do HTTP pokes. Not sure if the actual HTTP pokes work or not yet but the retry semantics are pretty good. --- synapse/http/client.py | 19 ++++++++++++ synapse/push/__init__.py | 58 ++++++++++++++++++++++++++++++------- synapse/push/httppusher.py | 55 ++++++++++++++++++++++++++++++++--- synapse/push/pusherpool.py | 8 +++-- synapse/storage/pusher.py | 26 ++++++++++++++--- synapse/storage/schema/delta/v7.sql | 2 ++ synapse/storage/schema/pusher.sql | 2 ++ 7 files changed, 150 insertions(+), 20 deletions(-) (limited to 'synapse') diff --git a/synapse/http/client.py b/synapse/http/client.py index 048a428905..82e80385ce 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -60,6 +60,25 @@ class SimpleHttpClient(object): defer.returnValue(json.loads(body)) + @defer.inlineCallbacks + def post_json_get_json(self, uri, post_json): + json_str = json.dumps(post_json) + + logger.info("HTTP POST %s -> %s", json_str, uri) + + response = yield self.agent.request( + "POST", + uri.encode("ascii"), + headers=Headers({ + "Content-Type": ["application/json"] + }), + bodyProducer=FileBodyProducer(StringIO(json_str)) + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + @defer.inlineCallbacks def get_json(self, uri, args={}): """ Get's some json from the given host and path diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index df0b91a8e9..a96f0f0183 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -26,12 +26,15 @@ logger = logging.getLogger(__name__) class Pusher(object): INITIAL_BACKOFF = 1000 - MAX_BACKOFF = 10 * 60 * 1000 + MAX_BACKOFF = 60 * 60 * 1000 + GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() + self.clock = self.hs.get_clock() self.user_name = user_name self.app = app self.app_display_name = app_display_name @@ -40,6 +43,7 @@ class Pusher(object): self.data = data self.last_token = last_token self.backoff_delay = Pusher.INITIAL_BACKOFF + self.failing_since = None @defer.inlineCallbacks def start(self): @@ -58,17 +62,51 @@ class Pusher(object): config = PaginationConfig(from_token=from_tok, limit='1') chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=100*365*24*60*60*1000) - if (self.dispatchPush(chunk['chunk'][0])): + # limiting to 1 may get 1 event plus 1 presence event, so pick out the actual event + singleEvent = None + for c in chunk['chunk']: + if 'event_id' in c: # Hmmm... + singleEvent = c + break + if not singleEvent: + continue + + ret = yield self.dispatchPush(singleEvent) + if (ret): self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + self.store.update_pusher_last_token_and_success(self.user_name, self.pushkey, + self.last_token, self.clock.time_msec()) + if self.failing_since: + self.failing_since = None + self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) else: - logger.warn("Failed to dispatch push for user %s. Trying again in %dms", - self.user_name, self.backoff_delay) - yield synapse.util.async.sleep(self.backoff_delay / 1000.0) - self.backoff_delay *=2 - if self.backoff_delay > Pusher.MAX_BACKOFF: - self.backoff_delay = Pusher.MAX_BACKOFF + if not self.failing_since: + self.failing_since = self.clock.time_msec() + self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + + if self.failing_since and self.failing_since < self.clock.time_msec() - Pusher.GIVE_UP_AFTER: + # we really only give up so that if the URL gets fixed, we don't suddenly deliver a load + # of old notifications. + logger.warn("Giving up on a notification to user %s, pushkey %s", + self.user_name, self.pushkey) + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + + self.failing_since = None + self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + else: + logger.warn("Failed to dispatch push for user %s (failing for %dms)." + "Trying again in %dms", + self.user_name, + self.clock.time_msec() - self.failing_since, + self.backoff_delay + ) + yield synapse.util.async.sleep(self.backoff_delay / 1000.0) + self.backoff_delay *=2 + if self.backoff_delay > Pusher.MAX_BACKOFF: + self.backoff_delay = Pusher.MAX_BACKOFF class PusherConfigException(Exception): diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index f3c3ca8191..33d735b974 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -14,13 +14,17 @@ # limitations under the License. from synapse.push import Pusher, PusherConfigException +from synapse.http.client import SimpleHttpClient + +from twisted.internet import defer import logging logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + last_token, last_success, failing_since): super(HttpPusher, self).__init__(_hs, user_name, app, @@ -28,12 +32,55 @@ class HttpPusher(Pusher): device_display_name, pushkey, data, - last_token) + last_token, + last_success, + failing_since) if 'url' not in data: raise PusherConfigException("'url' required in data for HTTP pusher") self.url = data['url'] + self.httpCli = SimpleHttpClient(self.hs) + self.data_minus_url = {} + self.data_minus_url.update(self.data) + del self.data_minus_url['url'] + + def _build_notification_dict(self, event): + # we probably do not want to push for every presence update + # (we may want to be able to set up notifications when specific + # people sign in, but we'd want to only deliver the pertinent ones) + # Actually, presence events will not get this far now because we + # need to filter them out in the main Pusher code. + if 'event_id' not in event: + return None + + return { + 'notification': { + 'transition' : 'new', # everything is new for now: we don't have read receipts + 'id': event['event_id'], + 'type': event['type'], + 'from': event['user_id'], + # we may have to fetch this over federation and we can't trust it anyway: is it worth it? + #'fromDisplayName': 'Steve Stevington' + }, + #'counts': { -- we don't mark messages as read yet so we have no way of knowing + # 'unread': 1, + # 'missedCalls': 2 + # }, + 'devices': { + self.pushkey: { + 'data' : self.data_minus_url + } + } + } + @defer.inlineCallbacks def dispatchPush(self, event): - print event - return True + notificationDict = self._build_notification_dict(event) + if not notificationDict: + defer.returnValue(True) + try: + yield self.httpCli.post_json_get_json(self.url, notificationDict) + except: + logger.exception("Failed to push %s ", self.url) + defer.returnValue(False) + defer.returnValue(True) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 436040f123..3fa5a4c4ff 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -45,7 +45,9 @@ class PusherPool: "device_display_name": device_display_name, "pushkey": pushkey, "data": data, - "last_token": None + "last_token": None, + "last_success": None, + "failing_since": None }) self._add_pusher_to_store(user_name, kind, app, app_display_name, device_display_name, pushkey, data) @@ -69,7 +71,9 @@ class PusherPool: device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], data=pusherdict['data'], - last_token=pusherdict['last_token'] + last_token=pusherdict['last_token'], + last_success=pusherdict['last_success'], + failing_since=pusherdict['failing_since'] ) else: raise PusherConfigException("Unknown pusher type '%s' for user %s" % diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 047a5f42d9..ce158c4b18 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -29,7 +29,8 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers_after_id(self, min_id): sql = ( - "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, last_token " + "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, " + "last_token, last_success, failing_since " "FROM pushers " "WHERE id > ?" ) @@ -46,8 +47,9 @@ class PusherStore(SQLBaseStore): "device_display_name": r[5], "pushkey": r[6], "data": r[7], - "last_token": r[8] - + "last_token": r[8], + "last_success": r[9], + "failing_since": r[10] } for r in rows ] @@ -79,6 +81,20 @@ class PusherStore(SQLBaseStore): {'last_token': last_token} ) + @defer.inlineCallbacks + def update_pusher_last_token_and_success(self, user_name, pushkey, last_token, last_success): + yield self._simple_update_one(PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token, 'last_success': last_success} + ) + + @defer.inlineCallbacks + def update_pusher_failing_since(self, user_name, pushkey, failing_since): + yield self._simple_update_one(PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'failing_since': failing_since} + ) + class PushersTable(Table): table_name = "pushers" @@ -92,7 +108,9 @@ class PushersTable(Table): "device_display_name", "pushkey", "data", - "last_token" + "last_token", + "last_success", + "failing_since" ] EntryType = collections.namedtuple("PusherEntry", fields) \ No newline at end of file diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql index 7f6852485d..e83f7e7436 100644 --- a/synapse/storage/schema/delta/v7.sql +++ b/synapse/storage/schema/delta/v7.sql @@ -23,6 +23,8 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey blob NOT NULL, data text, last_token TEXT, + last_success BIGINT, + failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (user_name, pushkey) ); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 7f6852485d..e83f7e7436 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -23,6 +23,8 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey blob NOT NULL, data text, last_token TEXT, + last_success BIGINT, + failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (user_name, pushkey) ); -- cgit 1.4.1 From 88af58d41d561f1d9f6bbbfb2a1e8bd00dbbe638 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 3 Dec 2014 13:37:02 +0000 Subject: Update to app_id / app_instance_id (partially) and mangle to be PEP8 compliant. --- synapse/push/__init__.py | 97 +++++++++++++++++++++++++------------ synapse/push/httppusher.py | 75 +++++++++++++++------------- synapse/push/pusherpool.py | 75 +++++++++++++++++----------- synapse/rest/pusher.py | 32 +++++++----- synapse/storage/pusher.py | 54 ++++++++++++--------- synapse/storage/schema/delta/v7.sql | 5 +- synapse/storage/schema/pusher.sql | 5 +- 7 files changed, 213 insertions(+), 130 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index a96f0f0183..5fca3bd772 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -24,90 +24,127 @@ import logging logger = logging.getLogger(__name__) + class Pusher(object): INITIAL_BACKOFF = 1000 MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + def __init__(self, _hs, user_name, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() self.user_name = user_name - self.app = app + self.app_id = app_id + self.app_instance_id = app_instance_id self.app_display_name = app_display_name self.device_display_name = device_display_name self.pushkey = pushkey self.data = data self.last_token = last_token + self.last_success = last_success # not actually used self.backoff_delay = Pusher.INITIAL_BACKOFF - self.failing_since = None + self.failing_since = failing_since @defer.inlineCallbacks def start(self): if not self.last_token: - # First-time setup: get a token to start from (we can't just start from no token, ie. 'now' - # because we need the result to be reproduceable in case we fail to dispatch the push) + # First-time setup: get a token to start from (we can't + # just start from no token, ie. 'now' + # because we need the result to be reproduceable in case + # we fail to dispatch the push) config = PaginationConfig(from_token=None, limit='1') - chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=0) + chunk = yield self.evStreamHandler.get_stream( + self.user_name, config, timeout=0) self.last_token = chunk['end'] - self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + self.store.update_pusher_last_token( + self.user_name, self.pushkey, self.last_token) logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) while True: from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') - chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=100*365*24*60*60*1000) + chunk = yield self.evStreamHandler.get_stream( + self.user_name, config, timeout=100*365*24*60*60*1000) - # limiting to 1 may get 1 event plus 1 presence event, so pick out the actual event - singleEvent = None + # limiting to 1 may get 1 event plus 1 presence event, so + # pick out the actual event + single_event = None for c in chunk['chunk']: - if 'event_id' in c: # Hmmm... - singleEvent = c + if 'event_id' in c: # Hmmm... + single_event = c break - if not singleEvent: + if not single_event: continue - ret = yield self.dispatchPush(singleEvent) - if (ret): + ret = yield self.dispatch_push(single_event) + if ret: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token_and_success(self.user_name, self.pushkey, - self.last_token, self.clock.time_msec()) + self.store.update_pusher_last_token_and_success( + self.user_name, + self.pushkey, + self.last_token, + self.clock.time_msec() + ) if self.failing_since: self.failing_since = None - self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + self.store.update_pusher_failing_since( + self.user_name, + self.pushkey, + self.failing_since) else: if not self.failing_since: self.failing_since = self.clock.time_msec() - self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + self.store.update_pusher_failing_since( + self.user_name, + self.pushkey, + self.failing_since + ) - if self.failing_since and self.failing_since < self.clock.time_msec() - Pusher.GIVE_UP_AFTER: - # we really only give up so that if the URL gets fixed, we don't suddenly deliver a load + if self.failing_since and \ + self.failing_since < \ + self.clock.time_msec() - Pusher.GIVE_UP_AFTER: + # we really only give up so that if the URL gets + # fixed, we don't suddenly deliver a load # of old notifications. - logger.warn("Giving up on a notification to user %s, pushkey %s", + logger.warn("Giving up on a notification to user %s, " + "pushkey %s", self.user_name, self.pushkey) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + self.store.update_pusher_last_token( + self.user_name, + self.pushkey, + self.last_token + ) self.failing_since = None - self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + self.store.update_pusher_failing_since( + self.user_name, + self.pushkey, + self.failing_since + ) else: - logger.warn("Failed to dispatch push for user %s (failing for %dms)." + logger.warn("Failed to dispatch push for user %s " + "(failing for %dms)." "Trying again in %dms", - self.user_name, - self.clock.time_msec() - self.failing_since, - self.backoff_delay - ) + self.user_name, + self.clock.time_msec() - self.failing_since, + self.backoff_delay + ) yield synapse.util.async.sleep(self.backoff_delay / 1000.0) - self.backoff_delay *=2 + self.backoff_delay *= 2 if self.backoff_delay > Pusher.MAX_BACKOFF: self.backoff_delay = Pusher.MAX_BACKOFF + def dispatch_push(self, p): + pass + class PusherConfigException(Exception): def __init__(self, msg): diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 33d735b974..fd7fe4e39c 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -22,21 +22,28 @@ import logging logger = logging.getLogger(__name__) + class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + def __init__(self, _hs, user_name, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): - super(HttpPusher, self).__init__(_hs, - user_name, - app, - app_display_name, - device_display_name, - pushkey, - data, - last_token, - last_success, - failing_since) + super(HttpPusher, self).__init__( + _hs, + user_name, + app_id, + app_instance_id, + app_display_name, + device_display_name, + pushkey, + data, + last_token, + last_success, + failing_since + ) if 'url' not in data: - raise PusherConfigException("'url' required in data for HTTP pusher") + raise PusherConfigException( + "'url' required in data for HTTP pusher" + ) self.url = data['url'] self.httpCli = SimpleHttpClient(self.hs) self.data_minus_url = {} @@ -53,34 +60,36 @@ class HttpPusher(Pusher): return None return { - 'notification': { - 'transition' : 'new', # everything is new for now: we don't have read receipts - 'id': event['event_id'], - 'type': event['type'], - 'from': event['user_id'], - # we may have to fetch this over federation and we can't trust it anyway: is it worth it? - #'fromDisplayName': 'Steve Stevington' - }, - #'counts': { -- we don't mark messages as read yet so we have no way of knowing - # 'unread': 1, - # 'missedCalls': 2 - # }, - 'devices': { - self.pushkey: { - 'data' : self.data_minus_url + 'notification': { + 'transition': 'new', + # everything is new for now: we don't have read receipts + 'id': event['event_id'], + 'type': event['type'], + 'from': event['user_id'], + # we may have to fetch this over federation and we + # can't trust it anyway: is it worth it? + #'fromDisplayName': 'Steve Stevington' + }, + #'counts': { -- we don't mark messages as read yet so + # we have no way of knowing + # 'unread': 1, + # 'missedCalls': 2 + # }, + 'devices': { + self.pushkey: { + 'data': self.data_minus_url } - } + } } @defer.inlineCallbacks - def dispatchPush(self, event): - notificationDict = self._build_notification_dict(event) - if not notificationDict: + def dispatch_push(self, event): + notification_dict = self._build_notification_dict(event) + if not notification_dict: defer.returnValue(True) try: - yield self.httpCli.post_json_get_json(self.url, notificationDict) + yield self.httpCli.post_json_get_json(self.url, notification_dict) except: logger.exception("Failed to push %s ", self.url) defer.returnValue(False) defer.returnValue(True) - diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 3fa5a4c4ff..045c36f3b7 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -34,13 +34,17 @@ class PusherPool: def start(self): self._pushers_added() - def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): - # we try to create the pusher just to validate the config: it will then get pulled out of the database, - # recreated, added and started: this means we have only one code path adding pushers. + def add_pusher(self, user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data): + # we try to create the pusher just to validate the config: it + # will then get pulled out of the database, + # recreated, added and started: this means we have only one + # code path adding pushers. self._create_pusher({ "user_name": user_name, "kind": kind, - "app": app, + "app_id": app_id, + "app_instance_id": app_instance_id, "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, @@ -49,42 +53,55 @@ class PusherPool: "last_success": None, "failing_since": None }) - self._add_pusher_to_store(user_name, kind, app, app_display_name, device_display_name, pushkey, data) + self._add_pusher_to_store(user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, + pushkey, data) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): - yield self.store.add_pusher(user_name=user_name, - kind=kind, - app=app, - app_display_name=app_display_name, - device_display_name=device_display_name, - pushkey=pushkey, - data=json.dumps(data)) + def _add_pusher_to_store(self, user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, + pushkey, data): + yield self.store.add_pusher( + user_name=user_name, + kind=kind, + app_id=app_id, + app_instance_id=app_instance_id, + app_display_name=app_display_name, + device_display_name=device_display_name, + pushkey=pushkey, + data=json.dumps(data) + ) self._pushers_added() def _create_pusher(self, pusherdict): if pusherdict['kind'] == 'http': - return HttpPusher(self.hs, - user_name=pusherdict['user_name'], - app=pusherdict['app'], - app_display_name=pusherdict['app_display_name'], - device_display_name=pusherdict['device_display_name'], - pushkey=pusherdict['pushkey'], - data=pusherdict['data'], - last_token=pusherdict['last_token'], - last_success=pusherdict['last_success'], - failing_since=pusherdict['failing_since'] - ) + return HttpPusher( + self.hs, + user_name=pusherdict['user_name'], + app_id=pusherdict['app_id'], + app_instance_id=pusherdict['app_instance_id'], + app_display_name=pusherdict['app_display_name'], + device_display_name=pusherdict['device_display_name'], + pushkey=pusherdict['pushkey'], + data=pusherdict['data'], + last_token=pusherdict['last_token'], + last_success=pusherdict['last_success'], + failing_since=pusherdict['failing_since'] + ) else: - raise PusherConfigException("Unknown pusher type '%s' for user %s" % - (pusherdict['kind'], pusherdict['user_name'])) + raise PusherConfigException( + "Unknown pusher type '%s' for user %s" % + (pusherdict['kind'], pusherdict['user_name']) + ) @defer.inlineCallbacks def _pushers_added(self): - pushers = yield self.store.get_all_pushers_after_id(self.last_pusher_started) + pushers = yield self.store.get_all_pushers_after_id( + self.last_pusher_started + ) for p in pushers: p['data'] = json.loads(p['data']) - if (len(pushers)): + if len(pushers): self.last_pusher_started = pushers[-1]['id'] self._start_pushers(pushers) @@ -95,4 +112,4 @@ class PusherPool: p = self._create_pusher(pusherdict) if p: self.pushers.append(p) - p.start() \ No newline at end of file + p.start() diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index 85d0d1c8cd..a39341cd8b 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -31,30 +31,37 @@ class PusherRestServlet(RestServlet): content = _parse_json(request) - reqd = ['kind', 'app', 'app_display_name', 'device_display_name', 'data'] + reqd = ['kind', 'app_id', 'app_instance_id', 'app_display_name', + 'device_display_name', 'data'] missing = [] for i in reqd: if i not in content: missing.append(i) if len(missing): - raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) + raise SynapseError(400, "Missing parameters: "+','.join(missing), + errcode=Codes.MISSING_PARAM) pusher_pool = self.hs.get_pusherpool() try: - pusher_pool.add_pusher(user_name=user.to_string(), - kind=content['kind'], - app=content['app'], - app_display_name=content['app_display_name'], - device_display_name=content['device_display_name'], - pushkey=pushkey, - data=content['data']) + pusher_pool.add_pusher( + user_name=user.to_string(), + kind=content['kind'], + app_id=content['app_id'], + app_instance_id=content['app_instance_id'], + app_display_name=content['app_display_name'], + device_display_name=content['device_display_name'], + pushkey=pushkey, + data=content['data'] + ) except PusherConfigException as pce: - raise SynapseError(400, "Config Error: "+pce.message, errcode=Codes.MISSING_PARAM) + raise SynapseError(400, "Config Error: "+pce.message, + errcode=Codes.MISSING_PARAM) defer.returnValue((200, {})) - def on_OPTIONS(self, request): - return (200, {}) + def on_OPTIONS(self, _): + return 200, {} + # XXX: C+ped from rest/room.py - surely this should be common? def _parse_json(request): @@ -67,5 +74,6 @@ def _parse_json(request): except ValueError: raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + def register_servlets(hs, http_server): PusherRestServlet(hs).register(http_server) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index ce158c4b18..a858e46f3b 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -25,11 +25,13 @@ import logging logger = logging.getLogger(__name__) + class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers_after_id(self, min_id): sql = ( - "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, " + "SELECT id, user_name, kind, app_id, app_instance_id," + "app_display_name, device_display_name, pushkey, data, " "last_token, last_success, failing_since " "FROM pushers " "WHERE id > ?" @@ -42,14 +44,15 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "app": r[3], - "app_display_name": r[4], - "device_display_name": r[5], - "pushkey": r[6], - "data": r[7], - "last_token": r[8], - "last_success": r[9], - "failing_since": r[10] + "app_id": r[3], + "app_instance_id": r[4], + "app_display_name": r[5], + "device_display_name": r[6], + "pushkey": r[7], + "data": r[8], + "last_token": r[9], + "last_success": r[10], + "failing_since": r[11] } for r in rows ] @@ -57,12 +60,14 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + def add_pusher(self, user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data): try: yield self._simple_insert(PushersTable.table_name, dict( user_name=user_name, kind=kind, - app=app, + app_id=app_id, + app_instance_id=app_instance_id, app_display_name=app_display_name, device_display_name=device_display_name, pushkey=pushkey, @@ -76,23 +81,27 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def update_pusher_last_token(self, user_name, pushkey, last_token): - yield self._simple_update_one(PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, - {'last_token': last_token} + yield self._simple_update_one( + PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token} ) @defer.inlineCallbacks - def update_pusher_last_token_and_success(self, user_name, pushkey, last_token, last_success): - yield self._simple_update_one(PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, - {'last_token': last_token, 'last_success': last_success} + def update_pusher_last_token_and_success(self, user_name, pushkey, + last_token, last_success): + yield self._simple_update_one( + PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token, 'last_success': last_success} ) @defer.inlineCallbacks def update_pusher_failing_since(self, user_name, pushkey, failing_since): - yield self._simple_update_one(PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, - {'failing_since': failing_since} + yield self._simple_update_one( + PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'failing_since': failing_since} ) @@ -103,7 +112,8 @@ class PushersTable(Table): "id", "user_name", "kind", - "app" + "app_id", + "app_instance_id", "app_display_name", "device_display_name", "pushkey", diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql index e83f7e7436..b60aeda756 100644 --- a/synapse/storage/schema/delta/v7.sql +++ b/synapse/storage/schema/delta/v7.sql @@ -17,11 +17,12 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, kind varchar(8) NOT NULL, - app varchar(64) NOT NULL, + app_id varchar(64) NOT NULL, + app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, - data text, + data blob, last_token TEXT, last_success BIGINT, failing_since BIGINT, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index e83f7e7436..b60aeda756 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -17,11 +17,12 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, kind varchar(8) NOT NULL, - app varchar(64) NOT NULL, + app_id varchar(64) NOT NULL, + app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, - data text, + data blob, last_token TEXT, last_success BIGINT, failing_since BIGINT, -- cgit 1.4.1 From 9728c305a34a1f9546d2ce0ef4c54352dc55a16d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:49:22 +0000 Subject: after a few rethinks, a working implementation of pushers. --- synapse/push/__init__.py | 12 ++++-- synapse/push/httppusher.py | 25 +++++------ synapse/push/pusherpool.py | 47 +++++++++++---------- synapse/rest/pusher.py | 13 +++--- synapse/storage/_base.py | 45 ++++++++++++++++++++ synapse/storage/pusher.py | 83 +++++++++++++++++++++++++------------ synapse/storage/schema/delta/v7.sql | 3 +- synapse/storage/schema/pusher.sql | 3 +- 8 files changed, 158 insertions(+), 73 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 5fca3bd772..5fe8719fe7 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -30,7 +30,7 @@ class Pusher(object): MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app_id, app_instance_id, + def __init__(self, _hs, user_name, app_id, app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): self.hs = _hs @@ -39,7 +39,6 @@ class Pusher(object): self.clock = self.hs.get_clock() self.user_name = user_name self.app_id = app_id - self.app_instance_id = app_instance_id self.app_display_name = app_display_name self.device_display_name = device_display_name self.pushkey = pushkey @@ -48,6 +47,7 @@ class Pusher(object): self.last_success = last_success # not actually used self.backoff_delay = Pusher.INITIAL_BACKOFF self.failing_since = failing_since + self.alive = True @defer.inlineCallbacks def start(self): @@ -65,7 +65,7 @@ class Pusher(object): logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) - while True: + while self.alive: from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') chunk = yield self.evStreamHandler.get_stream( @@ -81,6 +81,9 @@ class Pusher(object): if not single_event: continue + if not self.alive: + continue + ret = yield self.dispatch_push(single_event) if ret: self.backoff_delay = Pusher.INITIAL_BACKOFF @@ -142,6 +145,9 @@ class Pusher(object): if self.backoff_delay > Pusher.MAX_BACKOFF: self.backoff_delay = Pusher.MAX_BACKOFF + def stop(self): + self.alive = False + def dispatch_push(self, p): pass diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index fd7fe4e39c..f94f673391 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -24,14 +24,13 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app_id, app_instance_id, + def __init__(self, _hs, user_name, app_id, app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, user_name, app_id, - app_instance_id, app_display_name, device_display_name, pushkey, @@ -69,16 +68,18 @@ class HttpPusher(Pusher): # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? #'fromDisplayName': 'Steve Stevington' - }, - #'counts': { -- we don't mark messages as read yet so - # we have no way of knowing - # 'unread': 1, - # 'missedCalls': 2 - # }, - 'devices': { - self.pushkey: { - 'data': self.data_minus_url - } + #'counts': { -- we don't mark messages as read yet so + # we have no way of knowing + # 'unread': 1, + # 'missedCalls': 2 + # }, + 'devices': [ + { + 'app_id': self.app_id, + 'pushkey': self.pushkey, + 'data': self.data_minus_url + } + ] } } diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 045c36f3b7..d34ef3f6cf 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -24,17 +24,23 @@ import json logger = logging.getLogger(__name__) + class PusherPool: def __init__(self, _hs): self.hs = _hs self.store = self.hs.get_datastore() - self.pushers = [] + self.pushers = {} self.last_pusher_started = -1 + @defer.inlineCallbacks def start(self): - self._pushers_added() + pushers = yield self.store.get_all_pushers() + for p in pushers: + p['data'] = json.loads(p['data']) + self._start_pushers(pushers) - def add_pusher(self, user_name, kind, app_id, app_instance_id, + @defer.inlineCallbacks + def add_pusher(self, user_name, kind, app_id, app_display_name, device_display_name, pushkey, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, @@ -44,7 +50,6 @@ class PusherPool: "user_name": user_name, "kind": kind, "app_id": app_id, - "app_instance_id": app_instance_id, "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, @@ -53,25 +58,26 @@ class PusherPool: "last_success": None, "failing_since": None }) - self._add_pusher_to_store(user_name, kind, app_id, app_instance_id, - app_display_name, device_display_name, - pushkey, data) + yield self._add_pusher_to_store( + user_name, kind, app_id, + app_display_name, device_display_name, + pushkey, data + ) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, kind, app_id, app_instance_id, + def _add_pusher_to_store(self, user_name, kind, app_id, app_display_name, device_display_name, pushkey, data): yield self.store.add_pusher( user_name=user_name, kind=kind, app_id=app_id, - app_instance_id=app_instance_id, app_display_name=app_display_name, device_display_name=device_display_name, pushkey=pushkey, data=json.dumps(data) ) - self._pushers_added() + self._refresh_pusher((app_id, pushkey)) def _create_pusher(self, pusherdict): if pusherdict['kind'] == 'http': @@ -79,7 +85,6 @@ class PusherPool: self.hs, user_name=pusherdict['user_name'], app_id=pusherdict['app_id'], - app_instance_id=pusherdict['app_instance_id'], app_display_name=pusherdict['app_display_name'], device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], @@ -95,21 +100,21 @@ class PusherPool: ) @defer.inlineCallbacks - def _pushers_added(self): - pushers = yield self.store.get_all_pushers_after_id( - self.last_pusher_started + def _refresh_pusher(self, app_id_pushkey): + p = yield self.store.get_pushers_by_app_id_and_pushkey( + app_id_pushkey ) - for p in pushers: - p['data'] = json.loads(p['data']) - if len(pushers): - self.last_pusher_started = pushers[-1]['id'] + p['data'] = json.loads(p['data']) - self._start_pushers(pushers) + self._start_pushers([p]) def _start_pushers(self, pushers): - logger.info("Starting %d pushers", (len(pushers))) + logger.info("Starting %d pushers", len(pushers)) for pusherdict in pushers: p = self._create_pusher(pusherdict) if p: - self.pushers.append(p) + fullid = "%s:%s" % (pusherdict['app_id'], pusherdict['pushkey']) + if fullid in self.pushers: + self.pushers[fullid].stop() + self.pushers[fullid] = p p.start() diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index a39341cd8b..5b371318d0 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -23,16 +23,16 @@ import json class PusherRestServlet(RestServlet): - PATTERN = client_path_pattern("/pushers/(?P[\w]*)$") + PATTERN = client_path_pattern("/pushers/set$") @defer.inlineCallbacks - def on_PUT(self, request, pushkey): + def on_POST(self, request): user = yield self.auth.get_user_by_req(request) content = _parse_json(request) - reqd = ['kind', 'app_id', 'app_instance_id', 'app_display_name', - 'device_display_name', 'data'] + reqd = ['kind', 'app_id', 'app_display_name', + 'device_display_name', 'pushkey', 'data'] missing = [] for i in reqd: if i not in content: @@ -43,14 +43,13 @@ class PusherRestServlet(RestServlet): pusher_pool = self.hs.get_pusherpool() try: - pusher_pool.add_pusher( + yield pusher_pool.add_pusher( user_name=user.to_string(), kind=content['kind'], app_id=content['app_id'], - app_instance_id=content['app_instance_id'], app_display_name=content['app_display_name'], device_display_name=content['device_display_name'], - pushkey=pushkey, + pushkey=content['pushkey'], data=content['data'] ) except PusherConfigException as pce: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4881f03368..eb8cc4a9f3 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -195,6 +195,51 @@ class SQLBaseStore(object): txn.execute(sql, values.values()) return txn.lastrowid + def _simple_upsert(self, table, keyvalues, values): + """ + :param table: The table to upsert into + :param keyvalues: Dict of the unique key tables and their new values + :param values: Dict of all the nonunique columns and their new values + :return: A deferred + """ + return self.runInteraction( + "_simple_upsert", + self._simple_upsert_txn, table, keyvalues, values + ) + + def _simple_upsert_txn(self, txn, table, keyvalues, values): + # Try to update + sql = "UPDATE %s SET %s WHERE %s" % ( + table, + ", ".join("%s = ?" % (k) for k in values), + " AND ".join("%s = ?" % (k) for k in keyvalues) + ) + sqlargs = values.values() + keyvalues.values() + logger.debug( + "[SQL] %s Args=%s", + sql, sqlargs, + ) + + txn.execute(sql, sqlargs) + if txn.rowcount == 0: + # We didn't update and rows so insert a new one + allvalues = {} + allvalues.update(keyvalues) + allvalues.update(values) + + sql = "INSERT INTO %s (%s) VALUES (%s)" % ( + table, + ", ".join(k for k in allvalues), + ", ".join("?" for _ in allvalues) + ) + logger.debug( + "[SQL] %s Args=%s", + sql, keyvalues.values(), + ) + txn.execute(sql, allvalues.values()) + + + def _simple_select_one(self, table, keyvalues, retcols, allow_none=False): """Executes a SELECT query on the named table, which is expected to diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index a858e46f3b..deabd9cd2e 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -28,16 +28,48 @@ logger = logging.getLogger(__name__) class PusherStore(SQLBaseStore): @defer.inlineCallbacks - def get_all_pushers_after_id(self, min_id): + def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( - "SELECT id, user_name, kind, app_id, app_instance_id," + "SELECT id, user_name, kind, app_id," "app_display_name, device_display_name, pushkey, data, " "last_token, last_success, failing_since " "FROM pushers " - "WHERE id > ?" + "WHERE app_id = ? AND pushkey = ?" ) - rows = yield self._execute(None, sql, min_id) + rows = yield self._execute( + None, sql, app_id_and_pushkey[0], app_id_and_pushkey[1] + ) + + ret = [ + { + "id": r[0], + "user_name": r[1], + "kind": r[2], + "app_id": r[3], + "app_display_name": r[4], + "device_display_name": r[5], + "pushkey": r[6], + "data": r[7], + "last_token": r[8], + "last_success": r[9], + "failing_since": r[10] + } + for r in rows + ] + + defer.returnValue(ret[0]) + + @defer.inlineCallbacks + def get_all_pushers(self): + sql = ( + "SELECT id, user_name, kind, app_id," + "app_display_name, device_display_name, pushkey, data, " + "last_token, last_success, failing_since " + "FROM pushers" + ) + + rows = yield self._execute(None, sql) ret = [ { @@ -45,14 +77,13 @@ class PusherStore(SQLBaseStore): "user_name": r[1], "kind": r[2], "app_id": r[3], - "app_instance_id": r[4], - "app_display_name": r[5], - "device_display_name": r[6], - "pushkey": r[7], - "data": r[8], - "last_token": r[9], - "last_success": r[10], - "failing_since": r[11] + "app_display_name": r[4], + "device_display_name": r[5], + "pushkey": r[6], + "data": r[7], + "last_token": r[8], + "last_success": r[9], + "failing_since": r[10] } for r in rows ] @@ -60,21 +91,22 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app_id, app_instance_id, + def add_pusher(self, user_name, kind, app_id, app_display_name, device_display_name, pushkey, data): try: - yield self._simple_insert(PushersTable.table_name, dict( - user_name=user_name, - kind=kind, - app_id=app_id, - app_instance_id=app_instance_id, - app_display_name=app_display_name, - device_display_name=device_display_name, - pushkey=pushkey, - data=data - )) - except IntegrityError: - raise StoreError(409, "Pushkey in use.") + yield self._simple_upsert( + PushersTable.table_name, + dict( + app_id=app_id, + pushkey=pushkey, + ), + dict( + user_name=user_name, + kind=kind, + app_display_name=app_display_name, + device_display_name=device_display_name, + data=data + )) except Exception as e: logger.error("create_pusher with failed: %s", e) raise StoreError(500, "Problem creating pusher.") @@ -113,7 +145,6 @@ class PushersTable(Table): "user_name", "kind", "app_id", - "app_instance_id", "app_display_name", "device_display_name", "pushkey", diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql index b60aeda756..799e48d780 100644 --- a/synapse/storage/schema/delta/v7.sql +++ b/synapse/storage/schema/delta/v7.sql @@ -18,7 +18,6 @@ CREATE TABLE IF NOT EXISTS pushers ( user_name TEXT NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, - app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, @@ -27,5 +26,5 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (user_name, pushkey) + UNIQUE (app_id, pushkey) ); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index b60aeda756..799e48d780 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -18,7 +18,6 @@ CREATE TABLE IF NOT EXISTS pushers ( user_name TEXT NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, - app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, @@ -27,5 +26,5 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (user_name, pushkey) + UNIQUE (app_id, pushkey) ); -- cgit 1.4.1 From fc7c5e9cd7e0b1e29984233249311abe5cf23735 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:51:29 +0000 Subject: Rename the pusher SQL delta to v9 which the next free one --- synapse/storage/schema/delta/v7.sql | 30 ------------------------------ synapse/storage/schema/delta/v9.sql | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 30 deletions(-) delete mode 100644 synapse/storage/schema/delta/v7.sql create mode 100644 synapse/storage/schema/delta/v9.sql (limited to 'synapse') diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql deleted file mode 100644 index 799e48d780..0000000000 --- a/synapse/storage/schema/delta/v7.sql +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright 2014 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- Push notification endpoints that users have configured -CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - kind varchar(8) NOT NULL, - app_id varchar(64) NOT NULL, - app_display_name varchar(64) NOT NULL, - device_display_name varchar(128) NOT NULL, - pushkey blob NOT NULL, - data blob, - last_token TEXT, - last_success BIGINT, - failing_since BIGINT, - FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (app_id, pushkey) -); diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql new file mode 100644 index 0000000000..799e48d780 --- /dev/null +++ b/synapse/storage/schema/delta/v9.sql @@ -0,0 +1,30 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + kind varchar(8) NOT NULL, + app_id varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + data blob, + last_token TEXT, + last_success BIGINT, + failing_since BIGINT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (app_id, pushkey) +); -- cgit 1.4.1 From 173264b656b480a2f3634f49e78fd6093633af56 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:53:10 +0000 Subject: ...and bump SCHEMA_VERSION --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 642e5e289e..348c3b259c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -69,7 +69,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 8 +SCHEMA_VERSION = 9 class _RollbackButIsFineException(Exception): -- cgit 1.4.1 From 4c7ad50f6e50b95dfa9e0961a504e2f0d5b6921a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:55:04 +0000 Subject: Thank you, pyflakes --- synapse/storage/pusher.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index deabd9cd2e..9b5170a5f7 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -18,7 +18,6 @@ import collections from ._base import SQLBaseStore, Table from twisted.internet import defer -from sqlite3 import IntegrityError from synapse.api.errors import StoreError import logging -- cgit 1.4.1 From afa953a29301dcae40606171ed4cdac90eefab63 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 15:11:06 +0000 Subject: schema version is now 10 --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/v10.sql | 30 ++++++++++++++++++++++++++++++ synapse/storage/schema/delta/v9.sql | 30 ------------------------------ 3 files changed, 31 insertions(+), 31 deletions(-) create mode 100644 synapse/storage/schema/delta/v10.sql delete mode 100644 synapse/storage/schema/delta/v9.sql (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 348c3b259c..ad1765e04d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -69,7 +69,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql new file mode 100644 index 0000000000..799e48d780 --- /dev/null +++ b/synapse/storage/schema/delta/v10.sql @@ -0,0 +1,30 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + kind varchar(8) NOT NULL, + app_id varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + data blob, + last_token TEXT, + last_success BIGINT, + failing_since BIGINT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (app_id, pushkey) +); diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql deleted file mode 100644 index 799e48d780..0000000000 --- a/synapse/storage/schema/delta/v9.sql +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright 2014 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- Push notification endpoints that users have configured -CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - kind varchar(8) NOT NULL, - app_id varchar(64) NOT NULL, - app_display_name varchar(64) NOT NULL, - device_display_name varchar(128) NOT NULL, - pushkey blob NOT NULL, - data blob, - last_token TEXT, - last_success BIGINT, - failing_since BIGINT, - FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (app_id, pushkey) -); -- cgit 1.4.1 From fead431c181918558207f5115bd678a8984ce8d5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 18:44:33 +0000 Subject: If we didn't get any events, advance the token or we'll just keep not getting the same events again. --- synapse/push/__init__.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 5fe8719fe7..c5586be7b9 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -79,6 +79,7 @@ class Pusher(object): single_event = c break if not single_event: + self.last_token = chunk['end'] continue if not self.alive: -- cgit 1.4.1 From 70d0a453f353bf350e02d5306a98126f6b318c88 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 13 Jan 2015 13:14:41 +0000 Subject: Split out function to decide whether to notify or a given event --- synapse/push/__init__.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index c5586be7b9..f4795d559c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -49,6 +49,17 @@ class Pusher(object): self.failing_since = failing_since self.alive = True + def _should_notify_for_event(self, ev): + """ + This should take into account notification settings that the user + has configured both globally and per-room when we have the ability + to do such things. + """ + if ev['user_id'] == self.user_name: + # let's assume you probably know about messages you sent yourself + return False + return True + @defer.inlineCallbacks def start(self): if not self.last_token: @@ -85,8 +96,12 @@ class Pusher(object): if not self.alive: continue - ret = yield self.dispatch_push(single_event) - if ret: + processed = False + if self._should_notify_for_event(single_event): + processed = yield self.dispatch_push(single_event) + else: + processed = True + if processed: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token_and_success( -- cgit 1.4.1 From 2cb30767fa5e428f82c6c3ebced15d568d671c3c Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 13 Jan 2015 19:48:37 +0000 Subject: Honour the 'rejected' return from push gateways Add a timestamp to push tokens so we know the last time they we got them from the device. Send it to the push gateways so it can determine whether its failure is more recent than the token. Stop and remove pushers that have been rejected. --- synapse/push/__init__.py | 37 +++++++++++++++++++++++++++++++++--- synapse/push/httppusher.py | 15 ++++++++++----- synapse/push/pusherpool.py | 12 ++++++++++++ synapse/storage/pusher.py | 34 ++++++++++++++++++++++----------- synapse/storage/schema/delta/v10.sql | 1 + synapse/storage/schema/pusher.sql | 1 + 6 files changed, 81 insertions(+), 19 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index f4795d559c..839f666390 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -31,8 +31,8 @@ class Pusher(object): GIVE_UP_AFTER = 24 * 60 * 60 * 1000 def __init__(self, _hs, user_name, app_id, - app_display_name, device_display_name, pushkey, data, - last_token, last_success, failing_since): + app_display_name, device_display_name, pushkey, pushkey_ts, + data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() @@ -42,6 +42,7 @@ class Pusher(object): self.app_display_name = app_display_name self.device_display_name = device_display_name self.pushkey = pushkey + self.pushkey_ts = pushkey_ts self.data = data self.last_token = last_token self.last_success = last_success # not actually used @@ -98,9 +99,31 @@ class Pusher(object): processed = False if self._should_notify_for_event(single_event): - processed = yield self.dispatch_push(single_event) + rejected = yield self.dispatch_push(single_event) + if not rejected == False: + processed = True + for pk in rejected: + if pk != self.pushkey: + # for sanity, we only remove the pushkey if it + # was the one we actually sent... + logger.warn( + ("Ignoring rejected pushkey %s because we" + + "didn't send it"), (pk,) + ) + else: + logger.info( + "Pushkey %s was rejected: removing", + pk + ) + yield self.hs.get_pusherpool().remove_pusher( + self.app_id, pk + ) else: processed = True + + if not self.alive: + continue + if processed: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] @@ -165,6 +188,14 @@ class Pusher(object): self.alive = False def dispatch_push(self, p): + """ + Overridden by implementing classes to actually deliver the notification + :param p: The event to notify for as a single event from the event stream + :return: If the notification was delivered, an array containing any + pushkeys that were rejected by the push gateway. + False if the notification could not be delivered (ie. + should be retried). + """ pass diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index f94f673391..bcfa06e2ab 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -25,8 +25,8 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): def __init__(self, _hs, user_name, app_id, - app_display_name, device_display_name, pushkey, data, - last_token, last_success, failing_since): + app_display_name, device_display_name, pushkey, pushkey_ts, + data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, user_name, @@ -34,6 +34,7 @@ class HttpPusher(Pusher): app_display_name, device_display_name, pushkey, + pushkey_ts, data, last_token, last_success, @@ -77,6 +78,7 @@ class HttpPusher(Pusher): { 'app_id': self.app_id, 'pushkey': self.pushkey, + 'pushkeyTs': long(self.pushkey_ts / 1000), 'data': self.data_minus_url } ] @@ -87,10 +89,13 @@ class HttpPusher(Pusher): def dispatch_push(self, event): notification_dict = self._build_notification_dict(event) if not notification_dict: - defer.returnValue(True) + defer.returnValue([]) try: - yield self.httpCli.post_json_get_json(self.url, notification_dict) + resp = yield self.httpCli.post_json_get_json(self.url, notification_dict) except: logger.exception("Failed to push %s ", self.url) defer.returnValue(False) - defer.returnValue(True) + rejected = [] + if 'rejected' in resp: + rejected = resp['rejected'] + defer.returnValue(rejected) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index d34ef3f6cf..edddc3003e 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -53,6 +53,7 @@ class PusherPool: "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, + "pushkey_ts": self.hs.get_clock().time_msec(), "data": data, "last_token": None, "last_success": None, @@ -75,6 +76,7 @@ class PusherPool: app_display_name=app_display_name, device_display_name=device_display_name, pushkey=pushkey, + pushkey_ts=self.hs.get_clock().time_msec(), data=json.dumps(data) ) self._refresh_pusher((app_id, pushkey)) @@ -88,6 +90,7 @@ class PusherPool: app_display_name=pusherdict['app_display_name'], device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], + pushkey_ts=pusherdict['pushkey_ts'], data=pusherdict['data'], last_token=pusherdict['last_token'], last_success=pusherdict['last_success'], @@ -118,3 +121,12 @@ class PusherPool: self.pushers[fullid].stop() self.pushers[fullid] = p p.start() + + @defer.inlineCallbacks + def remove_pusher(self, app_id, pushkey): + fullid = "%s:%s" % (app_id, pushkey) + if fullid in self.pushers: + logger.info("Stopping pusher %s", fullid) + self.pushers[fullid].stop() + del self.pushers[fullid] + yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) \ No newline at end of file diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 9b5170a5f7..bfc4980256 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -30,7 +30,7 @@ class PusherStore(SQLBaseStore): def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( "SELECT id, user_name, kind, app_id," - "app_display_name, device_display_name, pushkey, data, " + "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers " "WHERE app_id = ? AND pushkey = ?" @@ -49,10 +49,11 @@ class PusherStore(SQLBaseStore): "app_display_name": r[4], "device_display_name": r[5], "pushkey": r[6], - "data": r[7], - "last_token": r[8], - "last_success": r[9], - "failing_since": r[10] + "pushkey_ts": r[7], + "data": r[8], + "last_token": r[9], + "last_success": r[10], + "failing_since": r[11] } for r in rows ] @@ -63,7 +64,7 @@ class PusherStore(SQLBaseStore): def get_all_pushers(self): sql = ( "SELECT id, user_name, kind, app_id," - "app_display_name, device_display_name, pushkey, data, " + "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers" ) @@ -79,10 +80,11 @@ class PusherStore(SQLBaseStore): "app_display_name": r[4], "device_display_name": r[5], "pushkey": r[6], - "data": r[7], - "last_token": r[8], - "last_success": r[9], - "failing_since": r[10] + "pushkey_ts": r[7], + "data": r[8], + "last_token": r[9], + "last_success": r[10], + "failing_since": r[11] } for r in rows ] @@ -91,7 +93,8 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def add_pusher(self, user_name, kind, app_id, - app_display_name, device_display_name, pushkey, data): + app_display_name, device_display_name, + pushkey, pushkey_ts, data): try: yield self._simple_upsert( PushersTable.table_name, @@ -104,12 +107,20 @@ class PusherStore(SQLBaseStore): kind=kind, app_display_name=app_display_name, device_display_name=device_display_name, + ts=pushkey_ts, data=data )) except Exception as e: logger.error("create_pusher with failed: %s", e) raise StoreError(500, "Problem creating pusher.") + @defer.inlineCallbacks + def delete_pusher_by_app_id_pushkey(self, app_id, pushkey): + yield self._simple_delete_one( + PushersTable.table_name, + dict(app_id=app_id, pushkey=pushkey) + ) + @defer.inlineCallbacks def update_pusher_last_token(self, user_name, pushkey, last_token): yield self._simple_update_one( @@ -147,6 +158,7 @@ class PushersTable(Table): "app_display_name", "device_display_name", "pushkey", + "pushkey_ts", "data", "last_token", "last_success", diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index 799e48d780..a991e4eb11 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -21,6 +21,7 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, + ts BIGINT NOT NULL, data blob, last_token TEXT, last_success BIGINT, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 799e48d780..a991e4eb11 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -21,6 +21,7 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, + ts BIGINT NOT NULL, data blob, last_token TEXT, last_success BIGINT, -- cgit 1.4.1 From e3e2fc3255665ac888f3e0c0c6e2be39d5fda5f3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 15 Jan 2015 16:17:21 +0000 Subject: Don't make the pushers' event streams cause people to appear online --- synapse/handlers/events.py | 43 ++++++++++++++++++++++--------------------- synapse/push/__init__.py | 4 +++- 2 files changed, 25 insertions(+), 22 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index c9ade253dd..54ab27004f 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -47,11 +47,11 @@ class EventStreamHandler(BaseHandler): @defer.inlineCallbacks @log_function def get_stream(self, auth_user_id, pagin_config, timeout=0, - as_client_event=True): + as_client_event=True, affect_presence=True): auth_user = self.hs.parse_userid(auth_user_id) try: - if auth_user not in self._streams_per_user: + if affect_presence and auth_user not in self._streams_per_user: self._streams_per_user[auth_user] = 0 if auth_user in self._stop_timer_per_user: try: @@ -64,7 +64,7 @@ class EventStreamHandler(BaseHandler): yield self.distributor.fire( "started_user_eventstream", auth_user ) - self._streams_per_user[auth_user] += 1 + self._streams_per_user[auth_user] += 1 if pagin_config.from_token is None: pagin_config.from_token = None @@ -92,27 +92,28 @@ class EventStreamHandler(BaseHandler): defer.returnValue(chunk) finally: - self._streams_per_user[auth_user] -= 1 - if not self._streams_per_user[auth_user]: - del self._streams_per_user[auth_user] - - # 10 seconds of grace to allow the client to reconnect again - # before we think they're gone - def _later(): - logger.debug( - "_later stopped_user_eventstream %s", auth_user - ) + if affect_presence: + self._streams_per_user[auth_user] -= 1 + if not self._streams_per_user[auth_user]: + del self._streams_per_user[auth_user] + + # 10 seconds of grace to allow the client to reconnect again + # before we think they're gone + def _later(): + logger.debug( + "_later stopped_user_eventstream %s", auth_user + ) - self._stop_timer_per_user.pop(auth_user, None) + self._stop_timer_per_user.pop(auth_user, None) - yield self.distributor.fire( - "stopped_user_eventstream", auth_user - ) + yield self.distributor.fire( + "stopped_user_eventstream", auth_user + ) - logger.debug("Scheduling _later: for %s", auth_user) - self._stop_timer_per_user[auth_user] = ( - self.clock.call_later(30, _later) - ) + logger.debug("Scheduling _later: for %s", auth_user) + self._stop_timer_per_user[auth_user] = ( + self.clock.call_later(30, _later) + ) class EventHandler(BaseHandler): diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 839f666390..9cf996fb80 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -81,7 +81,9 @@ class Pusher(object): from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') chunk = yield self.evStreamHandler.get_stream( - self.user_name, config, timeout=100*365*24*60*60*1000) + self.user_name, config, + timeout=100*365*24*60*60*1000, affect_presence=False + ) # limiting to 1 may get 1 event plus 1 presence event, so # pick out the actual event -- cgit 1.4.1 From 2ca2dbc82183f7dbe8c01694bf1c32a8c4c4b9de Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 15 Jan 2015 16:56:18 +0000 Subject: Send room name and first alias in notification poke. --- synapse/push/__init__.py | 13 +++++++++++++ synapse/push/httppusher.py | 16 +++++++++++++--- synapse/storage/__init__.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 9cf996fb80..5f4e833add 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -61,6 +61,19 @@ class Pusher(object): return False return True + @defer.inlineCallbacks + def get_context_for_event(self, ev): + name_aliases = yield self.store.get_room_name_and_aliases( + ev['room_id'] + ) + + ctx = {'aliases': name_aliases[1]} + if name_aliases[0] is not None: + ctx['name'] = name_aliases[0] + + defer.returnValue(ctx) + + @defer.inlineCallbacks def start(self): if not self.last_token: diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index bcfa06e2ab..7631a741fa 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -50,6 +50,7 @@ class HttpPusher(Pusher): self.data_minus_url.update(self.data) del self.data_minus_url['url'] + @defer.inlineCallbacks def _build_notification_dict(self, event): # we probably do not want to push for every presence update # (we may want to be able to set up notifications when specific @@ -57,9 +58,11 @@ class HttpPusher(Pusher): # Actually, presence events will not get this far now because we # need to filter them out in the main Pusher code. if 'event_id' not in event: - return None + defer.returnValue(None) + + ctx = yield self.get_context_for_event(event) - return { + d = { 'notification': { 'transition': 'new', # everything is new for now: we don't have read receipts @@ -85,9 +88,16 @@ class HttpPusher(Pusher): } } + if len(ctx['aliases']): + d['notification']['roomAlias'] = ctx['aliases'][0] + if 'name' in ctx: + d['notification']['roomName'] = ctx['name'] + + defer.returnValue(d) + @defer.inlineCallbacks def dispatch_push(self, event): - notification_dict = self._build_notification_dict(event) + notification_dict = yield self._build_notification_dict(event) if not notification_dict: defer.returnValue([]) try: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index fa7ad0eea8..191fe462a5 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -384,6 +384,41 @@ class DataStore(RoomMemberStore, RoomStore, events = yield self._parse_events(results) defer.returnValue(events) + @defer.inlineCallbacks + def get_room_name_and_aliases(self, room_id): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + sql += " AND (s.type = 'm.room.name' AND s.state_key = '')" + sql += " OR s.type = 'm.room.aliases'" + args = (room_id,) + + results = yield self._execute_and_decode(sql, *args) + + events = yield self._parse_events(results) + + name = None + aliases = [] + + for e in events: + if e.type == 'm.room.name': + name = e.content['name'] + elif e.type == 'm.room.aliases': + aliases.extend(e.content['aliases']) + + defer.returnValue((name, aliases)) + @defer.inlineCallbacks def _get_min_token(self): row = yield self._execute( -- cgit 1.4.1 From 2d2953cf5fce26625e56fc1abc230735d007ea1e Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 16 Jan 2015 11:24:10 +0000 Subject: Require device language when adding a pusher. Because this seems like it might be useful to do sooner rather than later. --- synapse/push/pusherpool.py | 8 +++++--- synapse/rest/pusher.py | 3 ++- synapse/storage/pusher.py | 3 ++- synapse/storage/schema/delta/v10.sql | 1 + synapse/storage/schema/pusher.sql | 1 + 5 files changed, 11 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index edddc3003e..8c77f4b668 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -41,7 +41,7 @@ class PusherPool: @defer.inlineCallbacks def add_pusher(self, user_name, kind, app_id, - app_display_name, device_display_name, pushkey, data): + app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, # recreated, added and started: this means we have only one @@ -54,6 +54,7 @@ class PusherPool: "device_display_name": device_display_name, "pushkey": pushkey, "pushkey_ts": self.hs.get_clock().time_msec(), + "lang": lang, "data": data, "last_token": None, "last_success": None, @@ -62,13 +63,13 @@ class PusherPool: yield self._add_pusher_to_store( user_name, kind, app_id, app_display_name, device_display_name, - pushkey, data + pushkey, lang, data ) @defer.inlineCallbacks def _add_pusher_to_store(self, user_name, kind, app_id, app_display_name, device_display_name, - pushkey, data): + pushkey, lang, data): yield self.store.add_pusher( user_name=user_name, kind=kind, @@ -77,6 +78,7 @@ class PusherPool: device_display_name=device_display_name, pushkey=pushkey, pushkey_ts=self.hs.get_clock().time_msec(), + lang=lang, data=json.dumps(data) ) self._refresh_pusher((app_id, pushkey)) diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index 5b371318d0..6b9a59adb6 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -32,7 +32,7 @@ class PusherRestServlet(RestServlet): content = _parse_json(request) reqd = ['kind', 'app_id', 'app_display_name', - 'device_display_name', 'pushkey', 'data'] + 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] for i in reqd: if i not in content: @@ -50,6 +50,7 @@ class PusherRestServlet(RestServlet): app_display_name=content['app_display_name'], device_display_name=content['device_display_name'], pushkey=content['pushkey'], + lang=content['lang'], data=content['data'] ) except PusherConfigException as pce: diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index bfc4980256..4eb30c7bdf 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -94,7 +94,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def add_pusher(self, user_name, kind, app_id, app_display_name, device_display_name, - pushkey, pushkey_ts, data): + pushkey, pushkey_ts, lang, data): try: yield self._simple_upsert( PushersTable.table_name, @@ -108,6 +108,7 @@ class PusherStore(SQLBaseStore): app_display_name=app_display_name, device_display_name=device_display_name, ts=pushkey_ts, + lang=lang, data=data )) except Exception as e: diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index a991e4eb11..689d2dff8b 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS pushers ( device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, ts BIGINT NOT NULL, + lang varchar(8), data blob, last_token TEXT, last_success BIGINT, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index a991e4eb11..689d2dff8b 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS pushers ( device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, ts BIGINT NOT NULL, + lang varchar(8), data blob, last_token TEXT, last_success BIGINT, -- cgit 1.4.1 From afb714f7bebf88ac27eac018cffa2078e2723310 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 20 Jan 2015 11:49:48 +0000 Subject: add instance_handles to pushers so we have a way to refer to them even if the push token changes. --- synapse/push/__init__.py | 3 ++- synapse/push/httppusher.py | 3 ++- synapse/push/pusherpool.py | 9 ++++--- synapse/rest/pusher.py | 3 ++- synapse/storage/pusher.py | 46 ++++++++++++++++++++---------------- synapse/storage/schema/delta/v10.sql | 1 + synapse/storage/schema/pusher.sql | 1 + 7 files changed, 39 insertions(+), 27 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 5f4e833add..3ee652f3bc 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -30,13 +30,14 @@ class Pusher(object): MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app_id, + def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() + self.instance_handle = instance_handle, self.user_name = user_name self.app_id = app_id self.app_display_name = app_display_name diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 7631a741fa..9a3e0be15e 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -24,11 +24,12 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app_id, + def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, + instance_handle, user_name, app_id, app_display_name, diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 8c77f4b668..2dfecf178b 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -40,7 +40,7 @@ class PusherPool: self._start_pushers(pushers) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app_id, + def add_pusher(self, user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, @@ -49,6 +49,7 @@ class PusherPool: self._create_pusher({ "user_name": user_name, "kind": kind, + "instance_handle": instance_handle, "app_id": app_id, "app_display_name": app_display_name, "device_display_name": device_display_name, @@ -61,17 +62,18 @@ class PusherPool: "failing_since": None }) yield self._add_pusher_to_store( - user_name, kind, app_id, + user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, lang, data ) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, kind, app_id, + def _add_pusher_to_store(self, user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): yield self.store.add_pusher( user_name=user_name, + instance_handle=instance_handle, kind=kind, app_id=app_id, app_display_name=app_display_name, @@ -87,6 +89,7 @@ class PusherPool: if pusherdict['kind'] == 'http': return HttpPusher( self.hs, + instance_handle=pusherdict['instance_handle'], user_name=pusherdict['user_name'], app_id=pusherdict['app_id'], app_display_name=pusherdict['app_display_name'], diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index 6b9a59adb6..4659c9b1d9 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -31,7 +31,7 @@ class PusherRestServlet(RestServlet): content = _parse_json(request) - reqd = ['kind', 'app_id', 'app_display_name', + reqd = ['instance_handle', 'kind', 'app_id', 'app_display_name', 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] for i in reqd: @@ -45,6 +45,7 @@ class PusherRestServlet(RestServlet): try: yield pusher_pool.add_pusher( user_name=user.to_string(), + instance_handle=content['instance_handle'], kind=content['kind'], app_id=content['app_id'], app_display_name=content['app_display_name'], diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 4eb30c7bdf..113cdc8a8e 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -29,7 +29,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( - "SELECT id, user_name, kind, app_id," + "SELECT id, user_name, kind, instance_handle, app_id," "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers " @@ -45,15 +45,16 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "app_id": r[3], - "app_display_name": r[4], - "device_display_name": r[5], - "pushkey": r[6], - "pushkey_ts": r[7], - "data": r[8], - "last_token": r[9], - "last_success": r[10], - "failing_since": r[11] + "instance_handle": r[3], + "app_id": r[4], + "app_display_name": r[5], + "device_display_name": r[6], + "pushkey": r[7], + "pushkey_ts": r[8], + "data": r[9], + "last_token": r[10], + "last_success": r[11], + "failing_since": r[12] } for r in rows ] @@ -63,7 +64,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers(self): sql = ( - "SELECT id, user_name, kind, app_id," + "SELECT id, user_name, kind, instance_handle, app_id," "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers" @@ -76,15 +77,16 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "app_id": r[3], - "app_display_name": r[4], - "device_display_name": r[5], - "pushkey": r[6], - "pushkey_ts": r[7], - "data": r[8], - "last_token": r[9], - "last_success": r[10], - "failing_since": r[11] + "instance_handle": r[3], + "app_id": r[4], + "app_display_name": r[5], + "device_display_name": r[6], + "pushkey": r[7], + "pushkey_ts": r[8], + "data": r[9], + "last_token": r[10], + "last_success": r[11], + "failing_since": r[12] } for r in rows ] @@ -92,7 +94,7 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app_id, + def add_pusher(self, user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: @@ -105,6 +107,7 @@ class PusherStore(SQLBaseStore): dict( user_name=user_name, kind=kind, + instance_handle=instance_handle, app_display_name=app_display_name, device_display_name=device_display_name, ts=pushkey_ts, @@ -155,6 +158,7 @@ class PushersTable(Table): "id", "user_name", "kind", + "instance_handle", "app_id", "app_display_name", "device_display_name", diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index 689d2dff8b..b84ce20ef3 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, + instance_handle varchar(32) NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 689d2dff8b..b84ce20ef3 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, + instance_handle varchar(32) NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, -- cgit 1.4.1 From 5d5932d493dc769e95472835b438769567ed550e Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 20 Jan 2015 11:52:08 +0000 Subject: use underscores everywhere, not camelcase. --- synapse/push/httppusher.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 9a3e0be15e..46433ad4a9 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -72,17 +72,17 @@ class HttpPusher(Pusher): 'from': event['user_id'], # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? - #'fromDisplayName': 'Steve Stevington' + #'from_display_name': 'Steve Stevington' #'counts': { -- we don't mark messages as read yet so # we have no way of knowing # 'unread': 1, - # 'missedCalls': 2 + # 'missed_calls': 2 # }, 'devices': [ { 'app_id': self.app_id, 'pushkey': self.pushkey, - 'pushkeyTs': long(self.pushkey_ts / 1000), + 'pushkey_ts': long(self.pushkey_ts / 1000), 'data': self.data_minus_url } ] -- cgit 1.4.1 From 6dcade97be7f1331063fd12ac85e61c6f2cf7dac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Jan 2015 16:27:04 +0000 Subject: Implement new state resolution algorithm --- synapse/state.py | 103 +++++++++---- tests/test_state.py | 428 +++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 428 insertions(+), 103 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 8144fa02b4..7d58a76ede 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.api.constants import EventTypes +from synapse.api.errors import AuthError from synapse.events.snapshot import EventContext from collections import namedtuple @@ -42,6 +43,8 @@ class StateHandler(object): def __init__(self, hs): self.store = hs.get_datastore() + # self.auth = hs.get_auth() + self.hs = hs @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): @@ -210,15 +213,22 @@ class StateHandler(object): else: prev_states = [] + auth_events = { + k: e for k, e in unconflicted_state.items() + if k[0] in (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) + } + try: - new_state = {} - new_state.update(unconflicted_state) - for key, events in conflicted_state.items(): - new_state[key] = self._resolve_state_events(events) + resolved_state = self._resolve_state_events( + conflicted_state, auth_events + ) except: logger.exception("Failed to resolve state") raise + new_state = unconflicted_state + new_state.update(resolved_state) + defer.returnValue((None, new_state, prev_states)) def _get_power_level_from_event_state(self, event, user_id): @@ -238,36 +248,65 @@ class StateHandler(object): return 0 @log_function - def _resolve_state_events(self, events): - curr_events = events + def _resolve_state_events(self, conflicted_state, auth_events): + resolved_state = {} + power_key = (EventTypes.PowerLevels, "") + if power_key in conflicted_state.items(): + power_levels = conflicted_state[power_key] + resolved_state[power_key] = self._resolve_auth_events(power_levels) + + auth_events.update(resolved_state) + + for key, events in conflicted_state.items(): + if key[0] == EventTypes.Member: + resolved_state[key] = self._resolve_auth_events( + events, + auth_events + ) - new_powers = [ - self._get_power_level_from_event_state(e, e.user_id) - for e in curr_events - ] + auth_events.update(resolved_state) - new_powers = [ - int(p) if p else 0 for p in new_powers - ] + for key, events in conflicted_state.items(): + if key not in resolved_state: + resolved_state[key] = self._resolve_normal_events( + events, auth_events + ) - max_power = max(new_powers) + return resolved_state - curr_events = [ - z[0] for z in zip(curr_events, new_powers) - if z[1] == max_power - ] + def _resolve_auth_events(self, events, auth_events): + reverse = [i for i in reversed(self._ordered_events(events))] - if not curr_events: - raise RuntimeError("Max didn't get a max?") - elif len(curr_events) == 1: - return curr_events[0] - - # TODO: For now, just choose the one with the largest event_id. - return ( - sorted( - curr_events, - key=lambda e: hashlib.sha1( - e.event_id + e.user_id + e.room_id + e.type - ).hexdigest() - )[0] - ) + auth_events = dict(auth_events) + + prev_event = reverse[0] + for event in reverse[1:]: + auth_events[(prev_event.type, prev_event.state_key)] = prev_event + try: + # FIXME: hs.get_auth() is bad style, but we need to do it to + # get around circular deps. + self.hs.get_auth().check(event, auth_events) + prev_event = event + except AuthError: + return prev_event + + return event + + def _resolve_normal_events(self, events, auth_events): + for event in self._ordered_events(events): + try: + # FIXME: hs.get_auth() is bad style, but we need to do it to + # get around circular deps. + self.hs.get_auth().check(event, auth_events) + return event + except AuthError as e: + pass + + # Oh dear. + return event + + def _ordered_events(self, events): + def key_func(e): + return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() + + return sorted(events, key=key_func) \ No newline at end of file diff --git a/tests/test_state.py b/tests/test_state.py index 98ad9e54cd..019e794aa2 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -16,11 +16,120 @@ from tests import unittest from twisted.internet import defer +from synapse.events import FrozenEvent +from synapse.api.auth import Auth +from synapse.api.constants import EventTypes, Membership from synapse.state import StateHandler from mock import Mock +_next_event_id = 1000 + + +def create_event(name=None, type=None, state_key=None, depth=2, event_id=None, + prev_events=[], **kwargs): + global _next_event_id + + if not event_id: + _next_event_id += 1 + event_id = str(_next_event_id) + + if not name: + if state_key is not None: + name = "<%s-%s, %s>" % (type, state_key, event_id,) + else: + name = "<%s, %s>" % (type, event_id,) + + d = { + "event_id": event_id, + "type": type, + "sender": "@user_id:example.com", + "room_id": "!room_id:example.com", + "depth": depth, + "prev_events": prev_events, + } + + if state_key is not None: + d["state_key"] = state_key + + d.update(kwargs) + + event = FrozenEvent(d) + + return event + + +class StateGroupStore(object): + def __init__(self): + self._event_to_state_group = {} + self._group_to_state = {} + + self._next_group = 1 + + def get_state_groups(self, event_ids): + groups = {} + for event_id in event_ids: + group = self._event_to_state_group.get(event_id) + if group: + groups[group] = self._group_to_state[group] + + return defer.succeed(groups) + + def store_state_groups(self, event, context): + if context.current_state is None: + return + + state_events = context.current_state + + if event.is_state(): + state_events[(event.type, event.state_key)] = event + + state_group = context.state_group + if not state_group: + state_group = self._next_group + self._next_group += 1 + + self._group_to_state[state_group] = state_events.values() + + self._event_to_state_group[event.event_id] = state_group + + +class DictObj(dict): + def __init__(self, **kwargs): + super(DictObj, self).__init__(kwargs) + self.__dict__ = self + + +class Graph(object): + def __init__(self, nodes, edges): + events = {} + clobbered = set(events.keys()) + + for event_id, fields in nodes.items(): + refs = edges.get(event_id) + if refs: + clobbered.difference_update(refs) + prev_events = [(r, {}) for r in refs] + else: + prev_events = [] + + events[event_id] = create_event( + event_id=event_id, + prev_events=prev_events, + **fields + ) + + self._leaves = clobbered + self._events = sorted(events.values(), key=lambda e: e.depth) + + def walk(self): + return iter(self._events) + + def get_leaves(self): + return (self._events[i] for i in self._leaves) + + class StateTestCase(unittest.TestCase): def setUp(self): self.store = Mock( @@ -29,20 +138,188 @@ class StateTestCase(unittest.TestCase): "add_event_hashes", ] ) - hs = Mock(spec=["get_datastore"]) + hs = Mock(spec=["get_datastore", "get_auth", "get_state_handler"]) hs.get_datastore.return_value = self.store + hs.get_state_handler.return_value = None + hs.get_auth.return_value = Auth(hs) self.state = StateHandler(hs) self.event_id = 0 + @defer.inlineCallbacks + def test_branch_no_conflict(self): + graph = Graph( + nodes={ + "START": DictObj( + type=EventTypes.Create, + state_key="", + depth=1, + ), + "A": DictObj( + type=EventTypes.Message, + depth=2, + ), + "B": DictObj( + type=EventTypes.Message, + depth=3, + ), + "C": DictObj( + type=EventTypes.Name, + state_key="", + depth=3, + ), + "D": DictObj( + type=EventTypes.Message, + depth=4, + ), + }, + edges={ + "A": ["START"], + "B": ["A"], + "C": ["A"], + "D": ["B", "C"] + } + ) + + store = StateGroupStore() + self.store.get_state_groups.side_effect = store.get_state_groups + + context_store = {} + + for event in graph.walk(): + context = yield self.state.compute_event_context(event) + store.store_state_groups(event, context) + context_store[event.event_id] = context + + self.assertEqual(2, len(context_store["D"].current_state)) + + @defer.inlineCallbacks + def test_branch_basic_conflict(self): + graph = Graph( + nodes={ + "START": DictObj( + type=EventTypes.Create, + state_key="creator", + content={"membership": "@user_id:example.com"}, + depth=1, + ), + "A": DictObj( + type=EventTypes.Member, + state_key="@user_id:example.com", + content={"membership": Membership.JOIN}, + membership=Membership.JOIN, + depth=2, + ), + "B": DictObj( + type=EventTypes.Name, + state_key="", + depth=3, + ), + "C": DictObj( + type=EventTypes.Name, + state_key="", + depth=4, + ), + "D": DictObj( + type=EventTypes.Message, + depth=5, + ), + }, + edges={ + "A": ["START"], + "B": ["A"], + "C": ["A"], + "D": ["B", "C"] + } + ) + + store = StateGroupStore() + self.store.get_state_groups.side_effect = store.get_state_groups + + context_store = {} + + for event in graph.walk(): + context = yield self.state.compute_event_context(event) + store.store_state_groups(event, context) + context_store[event.event_id] = context + + self.assertSetEqual( + {"START", "A", "C"}, + {e.event_id for e in context_store["D"].current_state.values()} + ) + + @defer.inlineCallbacks + def test_branch_have_banned_conflict(self): + graph = Graph( + nodes={ + "START": DictObj( + type=EventTypes.Create, + state_key="creator", + content={"membership": "@user_id:example.com"}, + depth=1, + ), + "A": DictObj( + type=EventTypes.Member, + state_key="@user_id:example.com", + content={"membership": Membership.JOIN}, + membership=Membership.JOIN, + depth=2, + ), + "B": DictObj( + type=EventTypes.Name, + state_key="", + depth=3, + ), + "C": DictObj( + type=EventTypes.Member, + state_key="@user_id_2:example.com", + content={"membership": Membership.BAN}, + membership=Membership.BAN, + depth=4, + ), + "D": DictObj( + type=EventTypes.Name, + state_key="", + depth=4, + sender="@user_id_2:example.com", + ), + "E": DictObj( + type=EventTypes.Message, + depth=5, + ), + }, + edges={ + "A": ["START"], + "B": ["A"], + "C": ["B"], + "D": ["B"], + "E": ["C", "D"] + } + ) + + store = StateGroupStore() + self.store.get_state_groups.side_effect = store.get_state_groups + + context_store = {} + + for event in graph.walk(): + context = yield self.state.compute_event_context(event) + store.store_state_groups(event, context) + context_store[event.event_id] = context + + self.assertSetEqual( + {"START", "A", "B", "C"}, + {e.event_id for e in context_store["E"].current_state.values()} + ) + @defer.inlineCallbacks def test_annotate_with_old_message(self): - event = self.create_event(type="test_message", name="event") + event = create_event(type="test_message", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] context = yield self.state.compute_event_context( @@ -62,12 +339,12 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_annotate_with_old_state(self): - event = self.create_event(type="state", state_key="", name="event") + event = create_event(type="state", state_key="", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] context = yield self.state.compute_event_context( @@ -88,13 +365,12 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_trivial_annotate_message(self): - event = self.create_event(type="test_message", name="event") - event.prev_events = [] + event = create_event(type="test_message", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] group_name = "group_name_1" @@ -119,13 +395,12 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_trivial_annotate_state(self): - event = self.create_event(type="state", state_key="", name="event") - event.prev_events = [] + event = create_event(type="state", state_key="", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] group_name = "group_name_1" @@ -150,30 +425,21 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_resolve_message_conflict(self): - event = self.create_event(type="test_message", name="event") - event.prev_events = [] + event = create_event(type="test_message", name="event") old_state_1 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] old_state_2 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test3", state_key="2"), - self.create_event(type="test4", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test3", state_key="2"), + create_event(type="test4", state_key=""), ] - group_name_1 = "group_name_1" - group_name_2 = "group_name_2" - - self.store.get_state_groups.return_value = { - group_name_1: old_state_1, - group_name_2: old_state_2, - } - - context = yield self.state.compute_event_context(event) + context = yield self._get_context(event, old_state_1, old_state_2) self.assertEqual(len(context.current_state), 5) @@ -181,56 +447,76 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_resolve_state_conflict(self): - event = self.create_event(type="test4", state_key="", name="event") - event.prev_events = [] + event = create_event(type="test4", state_key="", name="event") old_state_1 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] old_state_2 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test3", state_key="2"), - self.create_event(type="test4", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test3", state_key="2"), + create_event(type="test4", state_key=""), ] - group_name_1 = "group_name_1" - group_name_2 = "group_name_2" - - self.store.get_state_groups.return_value = { - group_name_1: old_state_1, - group_name_2: old_state_2, - } - - context = yield self.state.compute_event_context(event) + context = yield self._get_context(event, old_state_1, old_state_2) self.assertEqual(len(context.current_state), 5) self.assertIsNone(context.state_group) - def create_event(self, name=None, type=None, state_key=None): - self.event_id += 1 - event_id = str(self.event_id) + @defer.inlineCallbacks + def test_standard_depth_conflict(self): + event = create_event(type="test4", name="event") + + member_event = create_event( + type=EventTypes.Member, + state_key="@user_id:example.com", + content={ + "membership": Membership.JOIN, + } + ) - if not name: - if state_key is not None: - name = "<%s-%s>" % (type, state_key) - else: - name = "<%s>" % (type, ) + old_state_1 = [ + member_event, + create_event(type="test1", state_key="1", depth=1), + ] + + old_state_2 = [ + member_event, + create_event(type="test1", state_key="1", depth=2), + ] - event = Mock(name=name, spec=[]) - event.type = type + context = yield self._get_context(event, old_state_1, old_state_2) - if state_key is not None: - event.state_key = state_key - event.event_id = event_id + self.assertEqual(old_state_2[1], context.current_state[("test1", "1")]) + + # Reverse the depth to make sure we are actually using the depths + # during state resolution. + + old_state_1 = [ + member_event, + create_event(type="test1", state_key="1", depth=2), + ] + + old_state_2 = [ + member_event, + create_event(type="test1", state_key="1", depth=1), + ] + + context = yield self._get_context(event, old_state_1, old_state_2) + + self.assertEqual(old_state_1[1], context.current_state[("test1", "1")]) - event.is_state = lambda: (state_key is not None) - event.unsigned = {} + def _get_context(self, event, old_state_1, old_state_2): + group_name_1 = "group_name_1" + group_name_2 = "group_name_2" - event.user_id = "@user_id:example.com" - event.room_id = "!room_id:example.com" + self.store.get_state_groups.return_value = { + group_name_1: old_state_1, + group_name_2: old_state_2, + } - return event + return self.state.compute_event_context(event) -- cgit 1.4.1 From b390bf39f280c64497da089c647402bc3287f4b0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Jan 2015 16:44:04 +0000 Subject: Remove unused function. Add comment. --- synapse/state.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 7d58a76ede..5b622ad3b1 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -231,24 +231,20 @@ class StateHandler(object): defer.returnValue((None, new_state, prev_states)) - def _get_power_level_from_event_state(self, event, user_id): - if hasattr(event, "old_state_events") and event.old_state_events: - key = (EventTypes.PowerLevels, "", ) - power_level_event = event.old_state_events.get(key) - level = None - if power_level_event: - level = power_level_event.content.get("users", {}).get( - user_id - ) - if not level: - level = power_level_event.content.get("users_default", 0) - - return level - else: - return 0 - @log_function def _resolve_state_events(self, conflicted_state, auth_events): + """ This is where we actually decide which of the conflicted state to + use. + + We resolve conflicts in the following order: + 1. power levels + 2. memberships + 3. other events. + + :param conflicted_state: + :param auth_events: + :return: + """ resolved_state = {} power_key = (EventTypes.PowerLevels, "") if power_key in conflicted_state.items(): -- cgit 1.4.1 From 1d2016b4a881538aa86f4824f1131dfada186ae0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 22 Jan 2015 14:59:08 +0000 Subject: Move client v1 api rest servlets into a "client/v1" directory --- synapse/client/__init__.py | 14 + synapse/client/v1/__init__.py | 47 ++ synapse/client/v1/admin.py | 47 ++ synapse/client/v1/base.py | 80 +++ synapse/client/v1/directory.py | 112 ++++ synapse/client/v1/events.py | 81 +++ synapse/client/v1/initial_sync.py | 44 ++ synapse/client/v1/login.py | 109 ++++ synapse/client/v1/presence.py | 145 +++++ synapse/client/v1/profile.py | 113 ++++ synapse/client/v1/register.py | 291 ++++++++++ synapse/client/v1/room.py | 559 +++++++++++++++++++ synapse/client/v1/transactions.py | 95 ++++ synapse/client/v1/voip.py | 60 +++ synapse/rest/__init__.py | 47 -- synapse/rest/admin.py | 47 -- synapse/rest/base.py | 80 --- synapse/rest/directory.py | 112 ---- synapse/rest/events.py | 81 --- synapse/rest/initial_sync.py | 44 -- synapse/rest/login.py | 109 ---- synapse/rest/presence.py | 145 ----- synapse/rest/profile.py | 113 ---- synapse/rest/register.py | 291 ---------- synapse/rest/room.py | 559 ------------------- synapse/rest/transactions.py | 95 ---- synapse/rest/voip.py | 60 --- synapse/server.py | 2 +- tests/client/__init__.py | 14 + tests/client/v1/__init__.py | 15 + tests/client/v1/test_events.py | 225 ++++++++ tests/client/v1/test_presence.py | 372 +++++++++++++ tests/client/v1/test_profile.py | 150 ++++++ tests/client/v1/test_rooms.py | 1068 +++++++++++++++++++++++++++++++++++++ tests/client/v1/test_typing.py | 164 ++++++ tests/client/v1/utils.py | 134 +++++ tests/rest/__init__.py | 15 - tests/rest/test_events.py | 225 -------- tests/rest/test_presence.py | 372 ------------- tests/rest/test_profile.py | 150 ------ tests/rest/test_rooms.py | 1068 ------------------------------------- tests/rest/test_typing.py | 164 ------ tests/rest/utils.py | 134 ----- 43 files changed, 3940 insertions(+), 3912 deletions(-) create mode 100644 synapse/client/__init__.py create mode 100644 synapse/client/v1/__init__.py create mode 100644 synapse/client/v1/admin.py create mode 100644 synapse/client/v1/base.py create mode 100644 synapse/client/v1/directory.py create mode 100644 synapse/client/v1/events.py create mode 100644 synapse/client/v1/initial_sync.py create mode 100644 synapse/client/v1/login.py create mode 100644 synapse/client/v1/presence.py create mode 100644 synapse/client/v1/profile.py create mode 100644 synapse/client/v1/register.py create mode 100644 synapse/client/v1/room.py create mode 100644 synapse/client/v1/transactions.py create mode 100644 synapse/client/v1/voip.py delete mode 100644 synapse/rest/__init__.py delete mode 100644 synapse/rest/admin.py delete mode 100644 synapse/rest/base.py delete mode 100644 synapse/rest/directory.py delete mode 100644 synapse/rest/events.py delete mode 100644 synapse/rest/initial_sync.py delete mode 100644 synapse/rest/login.py delete mode 100644 synapse/rest/presence.py delete mode 100644 synapse/rest/profile.py delete mode 100644 synapse/rest/register.py delete mode 100644 synapse/rest/room.py delete mode 100644 synapse/rest/transactions.py delete mode 100644 synapse/rest/voip.py create mode 100644 tests/client/__init__.py create mode 100644 tests/client/v1/__init__.py create mode 100644 tests/client/v1/test_events.py create mode 100644 tests/client/v1/test_presence.py create mode 100644 tests/client/v1/test_profile.py create mode 100644 tests/client/v1/test_rooms.py create mode 100644 tests/client/v1/test_typing.py create mode 100644 tests/client/v1/utils.py delete mode 100644 tests/rest/__init__.py delete mode 100644 tests/rest/test_events.py delete mode 100644 tests/rest/test_presence.py delete mode 100644 tests/rest/test_profile.py delete mode 100644 tests/rest/test_rooms.py delete mode 100644 tests/rest/test_typing.py delete mode 100644 tests/rest/utils.py (limited to 'synapse') diff --git a/synapse/client/__init__.py b/synapse/client/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/synapse/client/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/client/v1/__init__.py b/synapse/client/v1/__init__.py new file mode 100644 index 0000000000..88ec9cd27d --- /dev/null +++ b/synapse/client/v1/__init__.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from . import ( + room, events, register, login, profile, presence, initial_sync, directory, + voip, admin, +) + + +class RestServletFactory(object): + + """ A factory for creating REST servlets. + + These REST servlets represent the entire client-server REST API. Generally + speaking, they serve as wrappers around events and the handlers that + process them. + + See synapse.events for information on synapse events. + """ + + def __init__(self, hs): + client_resource = hs.get_resource_for_client() + + # TODO(erikj): There *must* be a better way of doing this. + room.register_servlets(hs, client_resource) + events.register_servlets(hs, client_resource) + register.register_servlets(hs, client_resource) + login.register_servlets(hs, client_resource) + profile.register_servlets(hs, client_resource) + presence.register_servlets(hs, client_resource) + initial_sync.register_servlets(hs, client_resource) + directory.register_servlets(hs, client_resource) + voip.register_servlets(hs, client_resource) + admin.register_servlets(hs, client_resource) diff --git a/synapse/client/v1/admin.py b/synapse/client/v1/admin.py new file mode 100644 index 0000000000..0aa83514c8 --- /dev/null +++ b/synapse/client/v1/admin.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import AuthError, SynapseError +from base import RestServlet, client_path_pattern + +import logging + +logger = logging.getLogger(__name__) + + +class WhoisRestServlet(RestServlet): + PATTERN = client_path_pattern("/admin/whois/(?P[^/]*)") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + target_user = self.hs.parse_userid(user_id) + auth_user = yield self.auth.get_user_by_req(request) + is_admin = yield self.auth.is_server_admin(auth_user) + + if not is_admin and target_user != auth_user: + raise AuthError(403, "You are not a server admin") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only whois a local user") + + ret = yield self.handlers.admin_handler.get_whois(target_user) + + defer.returnValue((200, ret)) + + +def register_servlets(hs, http_server): + WhoisRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/base.py b/synapse/client/v1/base.py new file mode 100644 index 0000000000..d005206b77 --- /dev/null +++ b/synapse/client/v1/base.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains base REST classes for constructing REST servlets. """ +from synapse.api.urls import CLIENT_PREFIX +from .transactions import HttpTransactionStore +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def client_path_pattern(path_regex): + """Creates a regex compiled client path with the correct client path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + CLIENT_PREFIX + path_regex) + + +class RestServlet(object): + + """ A Synapse REST Servlet. + + An implementing class can either provide its own custom 'register' method, + or use the automatic pattern handling provided by the base class. + + To use this latter, the implementing class instead provides a `PATTERN` + class attribute containing a pre-compiled regular expression. The automatic + register method will then use this method to register any of the following + instance methods associated with the corresponding HTTP method: + + on_GET + on_PUT + on_POST + on_DELETE + on_OPTIONS + + Automatically handles turning CodeMessageExceptions thrown by these methods + into the appropriate HTTP response. + """ + + def __init__(self, hs): + self.hs = hs + + self.handlers = hs.get_handlers() + self.builder_factory = hs.get_event_builder_factory() + self.auth = hs.get_auth() + self.txns = HttpTransactionStore() + + def register(self, http_server): + """ Register this servlet with the given HTTP server. """ + if hasattr(self, "PATTERN"): + pattern = self.PATTERN + + for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"): + if hasattr(self, "on_%s" % (method)): + method_handler = getattr(self, "on_%s" % (method)) + http_server.register_path(method, pattern, method_handler) + else: + raise NotImplementedError("RestServlet must register something.") diff --git a/synapse/client/v1/directory.py b/synapse/client/v1/directory.py new file mode 100644 index 0000000000..7ff44fdd9e --- /dev/null +++ b/synapse/client/v1/directory.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from synapse.api.errors import AuthError, SynapseError, Codes +from base import RestServlet, client_path_pattern + +import json +import logging + + +logger = logging.getLogger(__name__) + + +def register_servlets(hs, http_server): + ClientDirectoryServer(hs).register(http_server) + + +class ClientDirectoryServer(RestServlet): + PATTERN = client_path_pattern("/directory/room/(?P[^/]*)$") + + @defer.inlineCallbacks + def on_GET(self, request, room_alias): + room_alias = self.hs.parse_roomalias(room_alias) + + dir_handler = self.handlers.directory_handler + res = yield dir_handler.get_association(room_alias) + + defer.returnValue((200, res)) + + @defer.inlineCallbacks + def on_PUT(self, request, room_alias): + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + if not "room_id" in content: + raise SynapseError(400, "Missing room_id key", + errcode=Codes.BAD_JSON) + + logger.debug("Got content: %s", content) + + room_alias = self.hs.parse_roomalias(room_alias) + + logger.debug("Got room name: %s", room_alias.to_string()) + + room_id = content["room_id"] + servers = content["servers"] if "servers" in content else None + + logger.debug("Got room_id: %s", room_id) + logger.debug("Got servers: %s", servers) + + # TODO(erikj): Check types. + # TODO(erikj): Check that room exists + + dir_handler = self.handlers.directory_handler + + try: + user_id = user.to_string() + yield dir_handler.create_association( + user_id, room_alias, room_id, servers + ) + yield dir_handler.send_room_alias_update_event(user_id, room_id) + except SynapseError as e: + raise e + except: + logger.exception("Failed to create association") + raise + + defer.returnValue((200, {})) + + @defer.inlineCallbacks + def on_DELETE(self, request, room_alias): + user = yield self.auth.get_user_by_req(request) + + is_admin = yield self.auth.is_server_admin(user) + if not is_admin: + raise AuthError(403, "You need to be a server admin") + + dir_handler = self.handlers.directory_handler + + room_alias = self.hs.parse_roomalias(room_alias) + + yield dir_handler.delete_association( + user.to_string(), room_alias + ) + + defer.returnValue((200, {})) + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) diff --git a/synapse/client/v1/events.py b/synapse/client/v1/events.py new file mode 100644 index 0000000000..c2515528ac --- /dev/null +++ b/synapse/client/v1/events.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains REST servlets to do with event streaming, /events.""" +from twisted.internet import defer + +from synapse.api.errors import SynapseError +from synapse.streams.config import PaginationConfig +from .base import RestServlet, client_path_pattern + +import logging + + +logger = logging.getLogger(__name__) + + +class EventStreamRestServlet(RestServlet): + PATTERN = client_path_pattern("/events$") + + DEFAULT_LONGPOLL_TIME_MS = 30000 + + @defer.inlineCallbacks + def on_GET(self, request): + auth_user = yield self.auth.get_user_by_req(request) + try: + handler = self.handlers.event_stream_handler + pagin_config = PaginationConfig.from_request(request) + timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS + if "timeout" in request.args: + try: + timeout = int(request.args["timeout"][0]) + except ValueError: + raise SynapseError(400, "timeout must be in milliseconds.") + + as_client_event = "raw" not in request.args + + chunk = yield handler.get_stream( + auth_user.to_string(), pagin_config, timeout=timeout, + as_client_event=as_client_event + ) + except: + logger.exception("Event stream failed") + raise + + defer.returnValue((200, chunk)) + + def on_OPTIONS(self, request): + return (200, {}) + + +# TODO: Unit test gets, with and without auth, with different kinds of events. +class EventRestServlet(RestServlet): + PATTERN = client_path_pattern("/events/(?P[^/]*)$") + + @defer.inlineCallbacks + def on_GET(self, request, event_id): + auth_user = yield self.auth.get_user_by_req(request) + handler = self.handlers.event_handler + event = yield handler.get_event(auth_user, event_id) + + if event: + defer.returnValue((200, self.hs.serialize_event(event))) + else: + defer.returnValue((404, "Event not found.")) + + +def register_servlets(hs, http_server): + EventStreamRestServlet(hs).register(http_server) + EventRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/initial_sync.py b/synapse/client/v1/initial_sync.py new file mode 100644 index 0000000000..b13d56b286 --- /dev/null +++ b/synapse/client/v1/initial_sync.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.streams.config import PaginationConfig +from base import RestServlet, client_path_pattern + + +# TODO: Needs unit testing +class InitialSyncRestServlet(RestServlet): + PATTERN = client_path_pattern("/initialSync$") + + @defer.inlineCallbacks + def on_GET(self, request): + user = yield self.auth.get_user_by_req(request) + with_feedback = "feedback" in request.args + as_client_event = "raw" not in request.args + pagination_config = PaginationConfig.from_request(request) + handler = self.handlers.message_handler + content = yield handler.snapshot_all_rooms( + user_id=user.to_string(), + pagin_config=pagination_config, + feedback=with_feedback, + as_client_event=as_client_event + ) + + defer.returnValue((200, content)) + + +def register_servlets(hs, http_server): + InitialSyncRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/login.py b/synapse/client/v1/login.py new file mode 100644 index 0000000000..6b8deff67b --- /dev/null +++ b/synapse/client/v1/login.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import SynapseError +from synapse.types import UserID +from base import RestServlet, client_path_pattern + +import json + + +class LoginRestServlet(RestServlet): + PATTERN = client_path_pattern("/login$") + PASS_TYPE = "m.login.password" + + def on_GET(self, request): + return (200, {"flows": [{"type": LoginRestServlet.PASS_TYPE}]}) + + def on_OPTIONS(self, request): + return (200, {}) + + @defer.inlineCallbacks + def on_POST(self, request): + login_submission = _parse_json(request) + try: + if login_submission["type"] == LoginRestServlet.PASS_TYPE: + result = yield self.do_password_login(login_submission) + defer.returnValue(result) + else: + raise SynapseError(400, "Bad login type.") + except KeyError: + raise SynapseError(400, "Missing JSON keys.") + + @defer.inlineCallbacks + def do_password_login(self, login_submission): + if not login_submission["user"].startswith('@'): + login_submission["user"] = UserID.create( + login_submission["user"], self.hs.hostname).to_string() + + handler = self.handlers.login_handler + token = yield handler.login( + user=login_submission["user"], + password=login_submission["password"]) + + result = { + "user_id": login_submission["user"], # may have changed + "access_token": token, + "home_server": self.hs.hostname, + } + + defer.returnValue((200, result)) + + +class LoginFallbackRestServlet(RestServlet): + PATTERN = client_path_pattern("/login/fallback$") + + def on_GET(self, request): + # TODO(kegan): This should be returning some HTML which is capable of + # hitting LoginRestServlet + return (200, {}) + + +class PasswordResetRestServlet(RestServlet): + PATTERN = client_path_pattern("/login/reset") + + @defer.inlineCallbacks + def on_POST(self, request): + reset_info = _parse_json(request) + try: + email = reset_info["email"] + user_id = reset_info["user_id"] + handler = self.handlers.login_handler + yield handler.reset_password(user_id, email) + # purposefully give no feedback to avoid people hammering different + # combinations. + defer.returnValue((200, {})) + except KeyError: + raise SynapseError( + 400, + "Missing keys. Requires 'email' and 'user_id'." + ) + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.") + return content + except ValueError: + raise SynapseError(400, "Content not JSON.") + + +def register_servlets(hs, http_server): + LoginRestServlet(hs).register(http_server) + # TODO PasswordResetRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/presence.py b/synapse/client/v1/presence.py new file mode 100644 index 0000000000..ca4d2d21f0 --- /dev/null +++ b/synapse/client/v1/presence.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains REST servlets to do with presence: /presence/ +""" +from twisted.internet import defer + +from synapse.api.errors import SynapseError +from base import RestServlet, client_path_pattern + +import json +import logging + +logger = logging.getLogger(__name__) + + +class PresenceStatusRestServlet(RestServlet): + PATTERN = client_path_pattern("/presence/(?P[^/]*)/status") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + state = yield self.handlers.presence_handler.get_state( + target_user=user, auth_user=auth_user) + + defer.returnValue((200, state)) + + @defer.inlineCallbacks + def on_PUT(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + state = {} + try: + content = json.loads(request.content.read()) + + state["presence"] = content.pop("presence") + + if "status_msg" in content: + state["status_msg"] = content.pop("status_msg") + if not isinstance(state["status_msg"], basestring): + raise SynapseError(400, "status_msg must be a string.") + + if content: + raise KeyError() + except SynapseError as e: + raise e + except: + raise SynapseError(400, "Unable to parse state") + + yield self.handlers.presence_handler.set_state( + target_user=user, auth_user=auth_user, state=state) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request): + return (200, {}) + + +class PresenceListRestServlet(RestServlet): + PATTERN = client_path_pattern("/presence/list/(?P[^/]*)") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + if not self.hs.is_mine(user): + raise SynapseError(400, "User not hosted on this Home Server") + + if auth_user != user: + raise SynapseError(400, "Cannot get another user's presence list") + + presence = yield self.handlers.presence_handler.get_presence_list( + observer_user=user, accepted=True) + + for p in presence: + observed_user = p.pop("observed_user") + p["user_id"] = observed_user.to_string() + + defer.returnValue((200, presence)) + + @defer.inlineCallbacks + def on_POST(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + if not self.hs.is_mine(user): + raise SynapseError(400, "User not hosted on this Home Server") + + if auth_user != user: + raise SynapseError( + 400, "Cannot modify another user's presence list") + + try: + content = json.loads(request.content.read()) + except: + logger.exception("JSON parse error") + raise SynapseError(400, "Unable to parse content") + + if "invite" in content: + for u in content["invite"]: + if not isinstance(u, basestring): + raise SynapseError(400, "Bad invite value.") + if len(u) == 0: + continue + invited_user = self.hs.parse_userid(u) + yield self.handlers.presence_handler.send_invite( + observer_user=user, observed_user=invited_user + ) + + if "drop" in content: + for u in content["drop"]: + if not isinstance(u, basestring): + raise SynapseError(400, "Bad drop value.") + if len(u) == 0: + continue + dropped_user = self.hs.parse_userid(u) + yield self.handlers.presence_handler.drop( + observer_user=user, observed_user=dropped_user + ) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request): + return (200, {}) + + +def register_servlets(hs, http_server): + PresenceStatusRestServlet(hs).register(http_server) + PresenceListRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/profile.py b/synapse/client/v1/profile.py new file mode 100644 index 0000000000..dc6eb424b0 --- /dev/null +++ b/synapse/client/v1/profile.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains REST servlets to do with profile: /profile/ """ +from twisted.internet import defer + +from base import RestServlet, client_path_pattern + +import json + + +class ProfileDisplaynameRestServlet(RestServlet): + PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + user = self.hs.parse_userid(user_id) + + displayname = yield self.handlers.profile_handler.get_displayname( + user, + ) + + defer.returnValue((200, {"displayname": displayname})) + + @defer.inlineCallbacks + def on_PUT(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + try: + content = json.loads(request.content.read()) + new_name = content["displayname"] + except: + defer.returnValue((400, "Unable to parse name")) + + yield self.handlers.profile_handler.set_displayname( + user, auth_user, new_name) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request, user_id): + return (200, {}) + + +class ProfileAvatarURLRestServlet(RestServlet): + PATTERN = client_path_pattern("/profile/(?P[^/]*)/avatar_url") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + user = self.hs.parse_userid(user_id) + + avatar_url = yield self.handlers.profile_handler.get_avatar_url( + user, + ) + + defer.returnValue((200, {"avatar_url": avatar_url})) + + @defer.inlineCallbacks + def on_PUT(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + try: + content = json.loads(request.content.read()) + new_name = content["avatar_url"] + except: + defer.returnValue((400, "Unable to parse name")) + + yield self.handlers.profile_handler.set_avatar_url( + user, auth_user, new_name) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request, user_id): + return (200, {}) + + +class ProfileRestServlet(RestServlet): + PATTERN = client_path_pattern("/profile/(?P[^/]*)") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + user = self.hs.parse_userid(user_id) + + displayname = yield self.handlers.profile_handler.get_displayname( + user, + ) + avatar_url = yield self.handlers.profile_handler.get_avatar_url( + user, + ) + + defer.returnValue((200, { + "displayname": displayname, + "avatar_url": avatar_url + })) + + +def register_servlets(hs, http_server): + ProfileDisplaynameRestServlet(hs).register(http_server) + ProfileAvatarURLRestServlet(hs).register(http_server) + ProfileRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/register.py b/synapse/client/v1/register.py new file mode 100644 index 0000000000..e3b26902d9 --- /dev/null +++ b/synapse/client/v1/register.py @@ -0,0 +1,291 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains REST servlets to do with registration: /register""" +from twisted.internet import defer + +from synapse.api.errors import SynapseError, Codes +from synapse.api.constants import LoginType +from base import RestServlet, client_path_pattern +import synapse.util.stringutils as stringutils + +from synapse.util.async import run_on_reactor + +from hashlib import sha1 +import hmac +import json +import logging +import urllib + +logger = logging.getLogger(__name__) + + +# We ought to be using hmac.compare_digest() but on older pythons it doesn't +# exist. It's a _really minor_ security flaw to use plain string comparison +# because the timing attack is so obscured by all the other code here it's +# unlikely to make much difference +if hasattr(hmac, "compare_digest"): + compare_digest = hmac.compare_digest +else: + compare_digest = lambda a, b: a == b + + +class RegisterRestServlet(RestServlet): + """Handles registration with the home server. + + This servlet is in control of the registration flow; the registration + handler doesn't have a concept of multi-stages or sessions. + """ + + PATTERN = client_path_pattern("/register$") + + def __init__(self, hs): + super(RegisterRestServlet, self).__init__(hs) + # sessions are stored as: + # self.sessions = { + # "session_id" : { __session_dict__ } + # } + # TODO: persistent storage + self.sessions = {} + + def on_GET(self, request): + if self.hs.config.enable_registration_captcha: + return ( + 200, + {"flows": [ + { + "type": LoginType.RECAPTCHA, + "stages": [ + LoginType.RECAPTCHA, + LoginType.EMAIL_IDENTITY, + LoginType.PASSWORD + ] + }, + { + "type": LoginType.RECAPTCHA, + "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD] + } + ]} + ) + else: + return ( + 200, + {"flows": [ + { + "type": LoginType.EMAIL_IDENTITY, + "stages": [ + LoginType.EMAIL_IDENTITY, LoginType.PASSWORD + ] + }, + { + "type": LoginType.PASSWORD + } + ]} + ) + + @defer.inlineCallbacks + def on_POST(self, request): + register_json = _parse_json(request) + + session = (register_json["session"] + if "session" in register_json else None) + login_type = None + if "type" not in register_json: + raise SynapseError(400, "Missing 'type' key.") + + try: + login_type = register_json["type"] + stages = { + LoginType.RECAPTCHA: self._do_recaptcha, + LoginType.PASSWORD: self._do_password, + LoginType.EMAIL_IDENTITY: self._do_email_identity + } + + session_info = self._get_session_info(request, session) + logger.debug("%s : session info %s request info %s", + login_type, session_info, register_json) + response = yield stages[login_type]( + request, + register_json, + session_info + ) + + if "access_token" not in response: + # isn't a final response + response["session"] = session_info["id"] + + defer.returnValue((200, response)) + except KeyError as e: + logger.exception(e) + raise SynapseError(400, "Missing JSON keys for login type %s." % ( + login_type, + )) + + def on_OPTIONS(self, request): + return (200, {}) + + def _get_session_info(self, request, session_id): + if not session_id: + # create a new session + while session_id is None or session_id in self.sessions: + session_id = stringutils.random_string(24) + self.sessions[session_id] = { + "id": session_id, + LoginType.EMAIL_IDENTITY: False, + LoginType.RECAPTCHA: False + } + + return self.sessions[session_id] + + def _save_session(self, session): + # TODO: Persistent storage + logger.debug("Saving session %s", session) + self.sessions[session["id"]] = session + + def _remove_session(self, session): + logger.debug("Removing session %s", session) + self.sessions.pop(session["id"]) + + @defer.inlineCallbacks + def _do_recaptcha(self, request, register_json, session): + if not self.hs.config.enable_registration_captcha: + raise SynapseError(400, "Captcha not required.") + + yield self._check_recaptcha(request, register_json, session) + + session[LoginType.RECAPTCHA] = True # mark captcha as done + self._save_session(session) + defer.returnValue({ + "next": [LoginType.PASSWORD, LoginType.EMAIL_IDENTITY] + }) + + @defer.inlineCallbacks + def _check_recaptcha(self, request, register_json, session): + if ("captcha_bypass_hmac" in register_json and + self.hs.config.captcha_bypass_secret): + if "user" not in register_json: + raise SynapseError(400, "Captcha bypass needs 'user'") + + want = hmac.new( + key=self.hs.config.captcha_bypass_secret, + msg=register_json["user"], + digestmod=sha1, + ).hexdigest() + + # str() because otherwise hmac complains that 'unicode' does not + # have the buffer interface + got = str(register_json["captcha_bypass_hmac"]) + + if compare_digest(want, got): + session["user"] = register_json["user"] + defer.returnValue(None) + else: + raise SynapseError( + 400, "Captcha bypass HMAC incorrect", + errcode=Codes.CAPTCHA_NEEDED + ) + + challenge = None + user_response = None + try: + challenge = register_json["challenge"] + user_response = register_json["response"] + except KeyError: + raise SynapseError(400, "Captcha response is required", + errcode=Codes.CAPTCHA_NEEDED) + + ip_addr = self.hs.get_ip_from_request(request) + + handler = self.handlers.registration_handler + yield handler.check_recaptcha( + ip_addr, + self.hs.config.recaptcha_private_key, + challenge, + user_response + ) + + @defer.inlineCallbacks + def _do_email_identity(self, request, register_json, session): + if (self.hs.config.enable_registration_captcha and + not session[LoginType.RECAPTCHA]): + raise SynapseError(400, "Captcha is required.") + + threepidCreds = register_json['threepidCreds'] + handler = self.handlers.registration_handler + logger.debug("Registering email. threepidcreds: %s" % (threepidCreds)) + yield handler.register_email(threepidCreds) + session["threepidCreds"] = threepidCreds # store creds for next stage + session[LoginType.EMAIL_IDENTITY] = True # mark email as done + self._save_session(session) + defer.returnValue({ + "next": LoginType.PASSWORD + }) + + @defer.inlineCallbacks + def _do_password(self, request, register_json, session): + yield run_on_reactor() + if (self.hs.config.enable_registration_captcha and + not session[LoginType.RECAPTCHA]): + # captcha should've been done by this stage! + raise SynapseError(400, "Captcha is required.") + + if ("user" in session and "user" in register_json and + session["user"] != register_json["user"]): + raise SynapseError( + 400, "Cannot change user ID during registration" + ) + + password = register_json["password"].encode("utf-8") + desired_user_id = (register_json["user"].encode("utf-8") + if "user" in register_json else None) + if (desired_user_id + and urllib.quote(desired_user_id) != desired_user_id): + raise SynapseError( + 400, + "User ID must only contain characters which do not " + + "require URL encoding.") + handler = self.handlers.registration_handler + (user_id, token) = yield handler.register( + localpart=desired_user_id, + password=password + ) + + if session[LoginType.EMAIL_IDENTITY]: + logger.debug("Binding emails %s to %s" % ( + session["threepidCreds"], user_id) + ) + yield handler.bind_emails(user_id, session["threepidCreds"]) + + result = { + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + } + self._remove_session(session) + defer.returnValue(result) + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.") + return content + except ValueError: + raise SynapseError(400, "Content not JSON.") + + +def register_servlets(hs, http_server): + RegisterRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/room.py b/synapse/client/v1/room.py new file mode 100644 index 0000000000..48bba2a5f3 --- /dev/null +++ b/synapse/client/v1/room.py @@ -0,0 +1,559 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains REST servlets to do with rooms: /rooms/ """ +from twisted.internet import defer + +from base import RestServlet, client_path_pattern +from synapse.api.errors import SynapseError, Codes +from synapse.streams.config import PaginationConfig +from synapse.api.constants import EventTypes, Membership + +import json +import logging +import urllib + + +logger = logging.getLogger(__name__) + + +class RoomCreateRestServlet(RestServlet): + # No PATTERN; we have custom dispatch rules here + + def register(self, http_server): + PATTERN = "/createRoom" + register_txn_path(self, PATTERN, http_server) + # define CORS for all of /rooms in RoomCreateRestServlet for simplicity + http_server.register_path("OPTIONS", + client_path_pattern("/rooms(?:/.*)?$"), + self.on_OPTIONS) + # define CORS for /createRoom[/txnid] + http_server.register_path("OPTIONS", + client_path_pattern("/createRoom(?:/.*)?$"), + self.on_OPTIONS) + + @defer.inlineCallbacks + def on_PUT(self, request, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + + response = yield self.on_POST(request) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + + @defer.inlineCallbacks + def on_POST(self, request): + auth_user = yield self.auth.get_user_by_req(request) + + room_config = self.get_room_config(request) + info = yield self.make_room(room_config, auth_user, None) + room_config.update(info) + defer.returnValue((200, info)) + + @defer.inlineCallbacks + def make_room(self, room_config, auth_user, room_id): + handler = self.handlers.room_creation_handler + info = yield handler.create_room( + user_id=auth_user.to_string(), + room_id=room_id, + config=room_config + ) + defer.returnValue(info) + + def get_room_config(self, request): + try: + user_supplied_config = json.loads(request.content.read()) + if "visibility" not in user_supplied_config: + # default visibility + user_supplied_config["visibility"] = "public" + return user_supplied_config + except (ValueError, TypeError): + raise SynapseError(400, "Body must be JSON.", + errcode=Codes.BAD_JSON) + + def on_OPTIONS(self, request): + return (200, {}) + + +# TODO: Needs unit testing for generic events +class RoomStateEventRestServlet(RestServlet): + def register(self, http_server): + # /room/$roomid/state/$eventtype + no_state_key = "/rooms/(?P[^/]*)/state/(?P[^/]*)$" + + # /room/$roomid/state/$eventtype/$statekey + state_key = ("/rooms/(?P[^/]*)/state/" + "(?P[^/]*)/(?P[^/]*)$") + + http_server.register_path("GET", + client_path_pattern(state_key), + self.on_GET) + http_server.register_path("PUT", + client_path_pattern(state_key), + self.on_PUT) + http_server.register_path("GET", + client_path_pattern(no_state_key), + self.on_GET_no_state_key) + http_server.register_path("PUT", + client_path_pattern(no_state_key), + self.on_PUT_no_state_key) + + def on_GET_no_state_key(self, request, room_id, event_type): + return self.on_GET(request, room_id, event_type, "") + + def on_PUT_no_state_key(self, request, room_id, event_type): + return self.on_PUT(request, room_id, event_type, "") + + @defer.inlineCallbacks + def on_GET(self, request, room_id, event_type, state_key): + user = yield self.auth.get_user_by_req(request) + + msg_handler = self.handlers.message_handler + data = yield msg_handler.get_room_data( + user_id=user.to_string(), + room_id=room_id, + event_type=event_type, + state_key=state_key, + ) + + if not data: + raise SynapseError( + 404, "Event not found.", errcode=Codes.NOT_FOUND + ) + defer.returnValue((200, data.get_dict()["content"])) + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, event_type, state_key): + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + event_dict = { + "type": event_type, + "content": content, + "room_id": room_id, + "sender": user.to_string(), + } + + if state_key is not None: + event_dict["state_key"] = state_key + + msg_handler = self.handlers.message_handler + yield msg_handler.create_and_send_event(event_dict) + + defer.returnValue((200, {})) + + +# TODO: Needs unit testing for generic events + feedback +class RoomSendEventRestServlet(RestServlet): + + def register(self, http_server): + # /rooms/$roomid/send/$event_type[/$txn_id] + PATTERN = ("/rooms/(?P[^/]*)/send/(?P[^/]*)") + register_txn_path(self, PATTERN, http_server, with_get=True) + + @defer.inlineCallbacks + def on_POST(self, request, room_id, event_type): + user = yield self.auth.get_user_by_req(request) + content = _parse_json(request) + + msg_handler = self.handlers.message_handler + event = yield msg_handler.create_and_send_event( + { + "type": event_type, + "content": content, + "room_id": room_id, + "sender": user.to_string(), + } + ) + + defer.returnValue((200, {"event_id": event.event_id})) + + def on_GET(self, request, room_id, event_type, txn_id): + return (200, "Not implemented") + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, event_type, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + + response = yield self.on_POST(request, room_id, event_type) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + + +# TODO: Needs unit testing for room ID + alias joins +class JoinRoomAliasServlet(RestServlet): + + def register(self, http_server): + # /join/$room_identifier[/$txn_id] + PATTERN = ("/join/(?P[^/]*)") + register_txn_path(self, PATTERN, http_server) + + @defer.inlineCallbacks + def on_POST(self, request, room_identifier): + user = yield self.auth.get_user_by_req(request) + + # the identifier could be a room alias or a room id. Try one then the + # other if it fails to parse, without swallowing other valid + # SynapseErrors. + + identifier = None + is_room_alias = False + try: + identifier = self.hs.parse_roomalias(room_identifier) + is_room_alias = True + except SynapseError: + identifier = self.hs.parse_roomid(room_identifier) + + # TODO: Support for specifying the home server to join with? + + if is_room_alias: + handler = self.handlers.room_member_handler + ret_dict = yield handler.join_room_alias(user, identifier) + defer.returnValue((200, ret_dict)) + else: # room id + msg_handler = self.handlers.message_handler + yield msg_handler.create_and_send_event( + { + "type": EventTypes.Member, + "content": {"membership": Membership.JOIN}, + "room_id": identifier.to_string(), + "sender": user.to_string(), + "state_key": user.to_string(), + } + ) + + defer.returnValue((200, {"room_id": identifier.to_string()})) + + @defer.inlineCallbacks + def on_PUT(self, request, room_identifier, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + + response = yield self.on_POST(request, room_identifier) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + + +# TODO: Needs unit testing +class PublicRoomListRestServlet(RestServlet): + PATTERN = client_path_pattern("/publicRooms$") + + @defer.inlineCallbacks + def on_GET(self, request): + handler = self.handlers.room_list_handler + data = yield handler.get_public_room_list() + defer.returnValue((200, data)) + + +# TODO: Needs unit testing +class RoomMemberListRestServlet(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/members$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + # TODO support Pagination stream API (limit/tokens) + user = yield self.auth.get_user_by_req(request) + handler = self.handlers.room_member_handler + members = yield handler.get_room_members_as_pagination_chunk( + room_id=room_id, + user_id=user.to_string()) + + for event in members["chunk"]: + # FIXME: should probably be state_key here, not user_id + target_user = self.hs.parse_userid(event["user_id"]) + # Presence is an optional cache; don't fail if we can't fetch it + try: + presence_handler = self.handlers.presence_handler + presence_state = yield presence_handler.get_state( + target_user=target_user, auth_user=user + ) + event["content"].update(presence_state) + except: + pass + + defer.returnValue((200, members)) + + +# TODO: Needs unit testing +class RoomMessageListRestServlet(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/messages$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + user = yield self.auth.get_user_by_req(request) + pagination_config = PaginationConfig.from_request( + request, default_limit=10, + ) + with_feedback = "feedback" in request.args + as_client_event = "raw" not in request.args + handler = self.handlers.message_handler + msgs = yield handler.get_messages( + room_id=room_id, + user_id=user.to_string(), + pagin_config=pagination_config, + feedback=with_feedback, + as_client_event=as_client_event + ) + + defer.returnValue((200, msgs)) + + +# TODO: Needs unit testing +class RoomStateRestServlet(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/state$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + user = yield self.auth.get_user_by_req(request) + handler = self.handlers.message_handler + # Get all the current state for this room + events = yield handler.get_state_events( + room_id=room_id, + user_id=user.to_string(), + ) + defer.returnValue((200, events)) + + +# TODO: Needs unit testing +class RoomInitialSyncRestServlet(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/initialSync$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + user = yield self.auth.get_user_by_req(request) + pagination_config = PaginationConfig.from_request(request) + content = yield self.handlers.message_handler.room_initial_sync( + room_id=room_id, + user_id=user.to_string(), + pagin_config=pagination_config, + ) + defer.returnValue((200, content)) + + +class RoomTriggerBackfill(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/backfill$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + remote_server = urllib.unquote( + request.args["remote"][0] + ).decode("UTF-8") + + limit = int(request.args["limit"][0]) + + handler = self.handlers.federation_handler + events = yield handler.backfill(remote_server, room_id, limit) + + res = [self.hs.serialize_event(event) for event in events] + defer.returnValue((200, res)) + + +# TODO: Needs unit testing +class RoomMembershipRestServlet(RestServlet): + + def register(self, http_server): + # /rooms/$roomid/[invite|join|leave] + PATTERN = ("/rooms/(?P[^/]*)/" + "(?Pjoin|invite|leave|ban|kick)") + register_txn_path(self, PATTERN, http_server) + + @defer.inlineCallbacks + def on_POST(self, request, room_id, membership_action): + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + # target user is you unless it is an invite + state_key = user.to_string() + if membership_action in ["invite", "ban", "kick"]: + if "user_id" not in content: + raise SynapseError(400, "Missing user_id key.") + state_key = content["user_id"] + + if membership_action == "kick": + membership_action = "leave" + + msg_handler = self.handlers.message_handler + yield msg_handler.create_and_send_event( + { + "type": EventTypes.Member, + "content": {"membership": unicode(membership_action)}, + "room_id": room_id, + "sender": user.to_string(), + "state_key": state_key, + } + ) + + defer.returnValue((200, {})) + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, membership_action, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + + response = yield self.on_POST(request, room_id, membership_action) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + + +class RoomRedactEventRestServlet(RestServlet): + def register(self, http_server): + PATTERN = ("/rooms/(?P[^/]*)/redact/(?P[^/]*)") + register_txn_path(self, PATTERN, http_server) + + @defer.inlineCallbacks + def on_POST(self, request, room_id, event_id): + user = yield self.auth.get_user_by_req(request) + content = _parse_json(request) + + msg_handler = self.handlers.message_handler + event = yield msg_handler.create_and_send_event( + { + "type": EventTypes.Redaction, + "content": content, + "room_id": room_id, + "sender": user.to_string(), + "redacts": event_id, + } + ) + + defer.returnValue((200, {"event_id": event.event_id})) + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, event_id, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + + response = yield self.on_POST(request, room_id, event_id) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + + +class RoomTypingRestServlet(RestServlet): + PATTERN = client_path_pattern( + "/rooms/(?P[^/]*)/typing/(?P[^/]*)$" + ) + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, user_id): + auth_user = yield self.auth.get_user_by_req(request) + + room_id = urllib.unquote(room_id) + target_user = self.hs.parse_userid(urllib.unquote(user_id)) + + content = _parse_json(request) + + typing_handler = self.handlers.typing_notification_handler + + if content["typing"]: + yield typing_handler.started_typing( + target_user=target_user, + auth_user=auth_user, + room_id=room_id, + timeout=content.get("timeout", 30000), + ) + else: + yield typing_handler.stopped_typing( + target_user=target_user, + auth_user=auth_user, + room_id=room_id, + ) + + defer.returnValue((200, {})) + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + + +def register_txn_path(servlet, regex_string, http_server, with_get=False): + """Registers a transaction-based path. + + This registers two paths: + PUT regex_string/$txnid + POST regex_string + + Args: + regex_string (str): The regex string to register. Must NOT have a + trailing $ as this string will be appended to. + http_server : The http_server to register paths with. + with_get: True to also register respective GET paths for the PUTs. + """ + http_server.register_path( + "POST", + client_path_pattern(regex_string + "$"), + servlet.on_POST + ) + http_server.register_path( + "PUT", + client_path_pattern(regex_string + "/(?P[^/]*)$"), + servlet.on_PUT + ) + if with_get: + http_server.register_path( + "GET", + client_path_pattern(regex_string + "/(?P[^/]*)$"), + servlet.on_GET + ) + + +def register_servlets(hs, http_server): + RoomStateEventRestServlet(hs).register(http_server) + RoomCreateRestServlet(hs).register(http_server) + RoomMemberListRestServlet(hs).register(http_server) + RoomMessageListRestServlet(hs).register(http_server) + JoinRoomAliasServlet(hs).register(http_server) + RoomTriggerBackfill(hs).register(http_server) + RoomMembershipRestServlet(hs).register(http_server) + RoomSendEventRestServlet(hs).register(http_server) + PublicRoomListRestServlet(hs).register(http_server) + RoomStateRestServlet(hs).register(http_server) + RoomInitialSyncRestServlet(hs).register(http_server) + RoomRedactEventRestServlet(hs).register(http_server) + RoomTypingRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/transactions.py b/synapse/client/v1/transactions.py new file mode 100644 index 0000000000..d933fea18a --- /dev/null +++ b/synapse/client/v1/transactions.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains logic for storing HTTP PUT transactions. This is used +to ensure idempotency when performing PUTs using the REST API.""" +import logging + +logger = logging.getLogger(__name__) + + +# FIXME: elsewhere we use FooStore to indicate something in the storage layer... +class HttpTransactionStore(object): + + def __init__(self): + # { key : (txn_id, response) } + self.transactions = {} + + def get_response(self, key, txn_id): + """Retrieve a response for this request. + + Args: + key (str): A transaction-independent key for this request. Usually + this is a combination of the path (without the transaction id) + and the user's access token. + txn_id (str): The transaction ID for this request + Returns: + A tuple of (HTTP response code, response content) or None. + """ + try: + logger.debug("get_response Key: %s TxnId: %s", key, txn_id) + (last_txn_id, response) = self.transactions[key] + if txn_id == last_txn_id: + logger.info("get_response: Returning a response for %s", key) + return response + except KeyError: + pass + return None + + def store_response(self, key, txn_id, response): + """Stores an HTTP response tuple. + + Args: + key (str): A transaction-independent key for this request. Usually + this is a combination of the path (without the transaction id) + and the user's access token. + txn_id (str): The transaction ID for this request. + response (tuple): A tuple of (HTTP response code, response content) + """ + logger.debug("store_response Key: %s TxnId: %s", key, txn_id) + self.transactions[key] = (txn_id, response) + + def store_client_transaction(self, request, txn_id, response): + """Stores the request/response pair of an HTTP transaction. + + Args: + request (twisted.web.http.Request): The twisted HTTP request. This + request must have the transaction ID as the last path segment. + response (tuple): A tuple of (response code, response dict) + txn_id (str): The transaction ID for this request. + """ + self.store_response(self._get_key(request), txn_id, response) + + def get_client_transaction(self, request, txn_id): + """Retrieves a stored response if there was one. + + Args: + request (twisted.web.http.Request): The twisted HTTP request. This + request must have the transaction ID as the last path segment. + txn_id (str): The transaction ID for this request. + Returns: + The response tuple. + Raises: + KeyError if the transaction was not found. + """ + response = self.get_response(self._get_key(request), txn_id) + if response is None: + raise KeyError("Transaction not found.") + return response + + def _get_key(self, request): + token = request.args["access_token"][0] + path_without_txn_id = request.path.rsplit("/", 1)[0] + return path_without_txn_id + "/" + token diff --git a/synapse/client/v1/voip.py b/synapse/client/v1/voip.py new file mode 100644 index 0000000000..011c35e69b --- /dev/null +++ b/synapse/client/v1/voip.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from base import RestServlet, client_path_pattern + + +import hmac +import hashlib +import base64 + + +class VoipRestServlet(RestServlet): + PATTERN = client_path_pattern("/voip/turnServer$") + + @defer.inlineCallbacks + def on_GET(self, request): + auth_user = yield self.auth.get_user_by_req(request) + + turnUris = self.hs.config.turn_uris + turnSecret = self.hs.config.turn_shared_secret + userLifetime = self.hs.config.turn_user_lifetime + if not turnUris or not turnSecret or not userLifetime: + defer.returnValue((200, {})) + + expiry = (self.hs.get_clock().time_msec() + userLifetime) / 1000 + username = "%d:%s" % (expiry, auth_user.to_string()) + + mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1) + # We need to use standard base64 encoding here, *not* syutil's + # encode_base64 because we need to add the standard padding to get the + # same result as the TURN server. + password = base64.b64encode(mac.digest()) + + defer.returnValue((200, { + 'username': username, + 'password': password, + 'ttl': userLifetime / 1000, + 'uris': turnUris, + })) + + def on_OPTIONS(self, request): + return (200, {}) + + +def register_servlets(hs, http_server): + VoipRestServlet(hs).register(http_server) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py deleted file mode 100644 index 88ec9cd27d..0000000000 --- a/synapse/rest/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from . import ( - room, events, register, login, profile, presence, initial_sync, directory, - voip, admin, -) - - -class RestServletFactory(object): - - """ A factory for creating REST servlets. - - These REST servlets represent the entire client-server REST API. Generally - speaking, they serve as wrappers around events and the handlers that - process them. - - See synapse.events for information on synapse events. - """ - - def __init__(self, hs): - client_resource = hs.get_resource_for_client() - - # TODO(erikj): There *must* be a better way of doing this. - room.register_servlets(hs, client_resource) - events.register_servlets(hs, client_resource) - register.register_servlets(hs, client_resource) - login.register_servlets(hs, client_resource) - profile.register_servlets(hs, client_resource) - presence.register_servlets(hs, client_resource) - initial_sync.register_servlets(hs, client_resource) - directory.register_servlets(hs, client_resource) - voip.register_servlets(hs, client_resource) - admin.register_servlets(hs, client_resource) diff --git a/synapse/rest/admin.py b/synapse/rest/admin.py deleted file mode 100644 index 0aa83514c8..0000000000 --- a/synapse/rest/admin.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from synapse.api.errors import AuthError, SynapseError -from base import RestServlet, client_path_pattern - -import logging - -logger = logging.getLogger(__name__) - - -class WhoisRestServlet(RestServlet): - PATTERN = client_path_pattern("/admin/whois/(?P[^/]*)") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - target_user = self.hs.parse_userid(user_id) - auth_user = yield self.auth.get_user_by_req(request) - is_admin = yield self.auth.is_server_admin(auth_user) - - if not is_admin and target_user != auth_user: - raise AuthError(403, "You are not a server admin") - - if not self.hs.is_mine(target_user): - raise SynapseError(400, "Can only whois a local user") - - ret = yield self.handlers.admin_handler.get_whois(target_user) - - defer.returnValue((200, ret)) - - -def register_servlets(hs, http_server): - WhoisRestServlet(hs).register(http_server) diff --git a/synapse/rest/base.py b/synapse/rest/base.py deleted file mode 100644 index c583945527..0000000000 --- a/synapse/rest/base.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" This module contains base REST classes for constructing REST servlets. """ -from synapse.api.urls import CLIENT_PREFIX -from synapse.rest.transactions import HttpTransactionStore -import re - -import logging - - -logger = logging.getLogger(__name__) - - -def client_path_pattern(path_regex): - """Creates a regex compiled client path with the correct client path - prefix. - - Args: - path_regex (str): The regex string to match. This should NOT have a ^ - as this will be prefixed. - Returns: - SRE_Pattern - """ - return re.compile("^" + CLIENT_PREFIX + path_regex) - - -class RestServlet(object): - - """ A Synapse REST Servlet. - - An implementing class can either provide its own custom 'register' method, - or use the automatic pattern handling provided by the base class. - - To use this latter, the implementing class instead provides a `PATTERN` - class attribute containing a pre-compiled regular expression. The automatic - register method will then use this method to register any of the following - instance methods associated with the corresponding HTTP method: - - on_GET - on_PUT - on_POST - on_DELETE - on_OPTIONS - - Automatically handles turning CodeMessageExceptions thrown by these methods - into the appropriate HTTP response. - """ - - def __init__(self, hs): - self.hs = hs - - self.handlers = hs.get_handlers() - self.builder_factory = hs.get_event_builder_factory() - self.auth = hs.get_auth() - self.txns = HttpTransactionStore() - - def register(self, http_server): - """ Register this servlet with the given HTTP server. """ - if hasattr(self, "PATTERN"): - pattern = self.PATTERN - - for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"): - if hasattr(self, "on_%s" % (method)): - method_handler = getattr(self, "on_%s" % (method)) - http_server.register_path(method, pattern, method_handler) - else: - raise NotImplementedError("RestServlet must register something.") diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py deleted file mode 100644 index 7ff44fdd9e..0000000000 --- a/synapse/rest/directory.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from twisted.internet import defer - -from synapse.api.errors import AuthError, SynapseError, Codes -from base import RestServlet, client_path_pattern - -import json -import logging - - -logger = logging.getLogger(__name__) - - -def register_servlets(hs, http_server): - ClientDirectoryServer(hs).register(http_server) - - -class ClientDirectoryServer(RestServlet): - PATTERN = client_path_pattern("/directory/room/(?P[^/]*)$") - - @defer.inlineCallbacks - def on_GET(self, request, room_alias): - room_alias = self.hs.parse_roomalias(room_alias) - - dir_handler = self.handlers.directory_handler - res = yield dir_handler.get_association(room_alias) - - defer.returnValue((200, res)) - - @defer.inlineCallbacks - def on_PUT(self, request, room_alias): - user = yield self.auth.get_user_by_req(request) - - content = _parse_json(request) - if not "room_id" in content: - raise SynapseError(400, "Missing room_id key", - errcode=Codes.BAD_JSON) - - logger.debug("Got content: %s", content) - - room_alias = self.hs.parse_roomalias(room_alias) - - logger.debug("Got room name: %s", room_alias.to_string()) - - room_id = content["room_id"] - servers = content["servers"] if "servers" in content else None - - logger.debug("Got room_id: %s", room_id) - logger.debug("Got servers: %s", servers) - - # TODO(erikj): Check types. - # TODO(erikj): Check that room exists - - dir_handler = self.handlers.directory_handler - - try: - user_id = user.to_string() - yield dir_handler.create_association( - user_id, room_alias, room_id, servers - ) - yield dir_handler.send_room_alias_update_event(user_id, room_id) - except SynapseError as e: - raise e - except: - logger.exception("Failed to create association") - raise - - defer.returnValue((200, {})) - - @defer.inlineCallbacks - def on_DELETE(self, request, room_alias): - user = yield self.auth.get_user_by_req(request) - - is_admin = yield self.auth.is_server_admin(user) - if not is_admin: - raise AuthError(403, "You need to be a server admin") - - dir_handler = self.handlers.directory_handler - - room_alias = self.hs.parse_roomalias(room_alias) - - yield dir_handler.delete_association( - user.to_string(), room_alias - ) - - defer.returnValue((200, {})) - - -def _parse_json(request): - try: - content = json.loads(request.content.read()) - if type(content) != dict: - raise SynapseError(400, "Content must be a JSON object.", - errcode=Codes.NOT_JSON) - return content - except ValueError: - raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) diff --git a/synapse/rest/events.py b/synapse/rest/events.py deleted file mode 100644 index bedcb2bcc6..0000000000 --- a/synapse/rest/events.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module contains REST servlets to do with event streaming, /events.""" -from twisted.internet import defer - -from synapse.api.errors import SynapseError -from synapse.streams.config import PaginationConfig -from synapse.rest.base import RestServlet, client_path_pattern - -import logging - - -logger = logging.getLogger(__name__) - - -class EventStreamRestServlet(RestServlet): - PATTERN = client_path_pattern("/events$") - - DEFAULT_LONGPOLL_TIME_MS = 30000 - - @defer.inlineCallbacks - def on_GET(self, request): - auth_user = yield self.auth.get_user_by_req(request) - try: - handler = self.handlers.event_stream_handler - pagin_config = PaginationConfig.from_request(request) - timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS - if "timeout" in request.args: - try: - timeout = int(request.args["timeout"][0]) - except ValueError: - raise SynapseError(400, "timeout must be in milliseconds.") - - as_client_event = "raw" not in request.args - - chunk = yield handler.get_stream( - auth_user.to_string(), pagin_config, timeout=timeout, - as_client_event=as_client_event - ) - except: - logger.exception("Event stream failed") - raise - - defer.returnValue((200, chunk)) - - def on_OPTIONS(self, request): - return (200, {}) - - -# TODO: Unit test gets, with and without auth, with different kinds of events. -class EventRestServlet(RestServlet): - PATTERN = client_path_pattern("/events/(?P[^/]*)$") - - @defer.inlineCallbacks - def on_GET(self, request, event_id): - auth_user = yield self.auth.get_user_by_req(request) - handler = self.handlers.event_handler - event = yield handler.get_event(auth_user, event_id) - - if event: - defer.returnValue((200, self.hs.serialize_event(event))) - else: - defer.returnValue((404, "Event not found.")) - - -def register_servlets(hs, http_server): - EventStreamRestServlet(hs).register(http_server) - EventRestServlet(hs).register(http_server) diff --git a/synapse/rest/initial_sync.py b/synapse/rest/initial_sync.py deleted file mode 100644 index b13d56b286..0000000000 --- a/synapse/rest/initial_sync.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from synapse.streams.config import PaginationConfig -from base import RestServlet, client_path_pattern - - -# TODO: Needs unit testing -class InitialSyncRestServlet(RestServlet): - PATTERN = client_path_pattern("/initialSync$") - - @defer.inlineCallbacks - def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) - with_feedback = "feedback" in request.args - as_client_event = "raw" not in request.args - pagination_config = PaginationConfig.from_request(request) - handler = self.handlers.message_handler - content = yield handler.snapshot_all_rooms( - user_id=user.to_string(), - pagin_config=pagination_config, - feedback=with_feedback, - as_client_event=as_client_event - ) - - defer.returnValue((200, content)) - - -def register_servlets(hs, http_server): - InitialSyncRestServlet(hs).register(http_server) diff --git a/synapse/rest/login.py b/synapse/rest/login.py deleted file mode 100644 index 6b8deff67b..0000000000 --- a/synapse/rest/login.py +++ /dev/null @@ -1,109 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from synapse.api.errors import SynapseError -from synapse.types import UserID -from base import RestServlet, client_path_pattern - -import json - - -class LoginRestServlet(RestServlet): - PATTERN = client_path_pattern("/login$") - PASS_TYPE = "m.login.password" - - def on_GET(self, request): - return (200, {"flows": [{"type": LoginRestServlet.PASS_TYPE}]}) - - def on_OPTIONS(self, request): - return (200, {}) - - @defer.inlineCallbacks - def on_POST(self, request): - login_submission = _parse_json(request) - try: - if login_submission["type"] == LoginRestServlet.PASS_TYPE: - result = yield self.do_password_login(login_submission) - defer.returnValue(result) - else: - raise SynapseError(400, "Bad login type.") - except KeyError: - raise SynapseError(400, "Missing JSON keys.") - - @defer.inlineCallbacks - def do_password_login(self, login_submission): - if not login_submission["user"].startswith('@'): - login_submission["user"] = UserID.create( - login_submission["user"], self.hs.hostname).to_string() - - handler = self.handlers.login_handler - token = yield handler.login( - user=login_submission["user"], - password=login_submission["password"]) - - result = { - "user_id": login_submission["user"], # may have changed - "access_token": token, - "home_server": self.hs.hostname, - } - - defer.returnValue((200, result)) - - -class LoginFallbackRestServlet(RestServlet): - PATTERN = client_path_pattern("/login/fallback$") - - def on_GET(self, request): - # TODO(kegan): This should be returning some HTML which is capable of - # hitting LoginRestServlet - return (200, {}) - - -class PasswordResetRestServlet(RestServlet): - PATTERN = client_path_pattern("/login/reset") - - @defer.inlineCallbacks - def on_POST(self, request): - reset_info = _parse_json(request) - try: - email = reset_info["email"] - user_id = reset_info["user_id"] - handler = self.handlers.login_handler - yield handler.reset_password(user_id, email) - # purposefully give no feedback to avoid people hammering different - # combinations. - defer.returnValue((200, {})) - except KeyError: - raise SynapseError( - 400, - "Missing keys. Requires 'email' and 'user_id'." - ) - - -def _parse_json(request): - try: - content = json.loads(request.content.read()) - if type(content) != dict: - raise SynapseError(400, "Content must be a JSON object.") - return content - except ValueError: - raise SynapseError(400, "Content not JSON.") - - -def register_servlets(hs, http_server): - LoginRestServlet(hs).register(http_server) - # TODO PasswordResetRestServlet(hs).register(http_server) diff --git a/synapse/rest/presence.py b/synapse/rest/presence.py deleted file mode 100644 index ca4d2d21f0..0000000000 --- a/synapse/rest/presence.py +++ /dev/null @@ -1,145 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" This module contains REST servlets to do with presence: /presence/ -""" -from twisted.internet import defer - -from synapse.api.errors import SynapseError -from base import RestServlet, client_path_pattern - -import json -import logging - -logger = logging.getLogger(__name__) - - -class PresenceStatusRestServlet(RestServlet): - PATTERN = client_path_pattern("/presence/(?P[^/]*)/status") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - state = yield self.handlers.presence_handler.get_state( - target_user=user, auth_user=auth_user) - - defer.returnValue((200, state)) - - @defer.inlineCallbacks - def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - state = {} - try: - content = json.loads(request.content.read()) - - state["presence"] = content.pop("presence") - - if "status_msg" in content: - state["status_msg"] = content.pop("status_msg") - if not isinstance(state["status_msg"], basestring): - raise SynapseError(400, "status_msg must be a string.") - - if content: - raise KeyError() - except SynapseError as e: - raise e - except: - raise SynapseError(400, "Unable to parse state") - - yield self.handlers.presence_handler.set_state( - target_user=user, auth_user=auth_user, state=state) - - defer.returnValue((200, {})) - - def on_OPTIONS(self, request): - return (200, {}) - - -class PresenceListRestServlet(RestServlet): - PATTERN = client_path_pattern("/presence/list/(?P[^/]*)") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - if not self.hs.is_mine(user): - raise SynapseError(400, "User not hosted on this Home Server") - - if auth_user != user: - raise SynapseError(400, "Cannot get another user's presence list") - - presence = yield self.handlers.presence_handler.get_presence_list( - observer_user=user, accepted=True) - - for p in presence: - observed_user = p.pop("observed_user") - p["user_id"] = observed_user.to_string() - - defer.returnValue((200, presence)) - - @defer.inlineCallbacks - def on_POST(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - if not self.hs.is_mine(user): - raise SynapseError(400, "User not hosted on this Home Server") - - if auth_user != user: - raise SynapseError( - 400, "Cannot modify another user's presence list") - - try: - content = json.loads(request.content.read()) - except: - logger.exception("JSON parse error") - raise SynapseError(400, "Unable to parse content") - - if "invite" in content: - for u in content["invite"]: - if not isinstance(u, basestring): - raise SynapseError(400, "Bad invite value.") - if len(u) == 0: - continue - invited_user = self.hs.parse_userid(u) - yield self.handlers.presence_handler.send_invite( - observer_user=user, observed_user=invited_user - ) - - if "drop" in content: - for u in content["drop"]: - if not isinstance(u, basestring): - raise SynapseError(400, "Bad drop value.") - if len(u) == 0: - continue - dropped_user = self.hs.parse_userid(u) - yield self.handlers.presence_handler.drop( - observer_user=user, observed_user=dropped_user - ) - - defer.returnValue((200, {})) - - def on_OPTIONS(self, request): - return (200, {}) - - -def register_servlets(hs, http_server): - PresenceStatusRestServlet(hs).register(http_server) - PresenceListRestServlet(hs).register(http_server) diff --git a/synapse/rest/profile.py b/synapse/rest/profile.py deleted file mode 100644 index dc6eb424b0..0000000000 --- a/synapse/rest/profile.py +++ /dev/null @@ -1,113 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" This module contains REST servlets to do with profile: /profile/ """ -from twisted.internet import defer - -from base import RestServlet, client_path_pattern - -import json - - -class ProfileDisplaynameRestServlet(RestServlet): - PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) - - displayname = yield self.handlers.profile_handler.get_displayname( - user, - ) - - defer.returnValue((200, {"displayname": displayname})) - - @defer.inlineCallbacks - def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - try: - content = json.loads(request.content.read()) - new_name = content["displayname"] - except: - defer.returnValue((400, "Unable to parse name")) - - yield self.handlers.profile_handler.set_displayname( - user, auth_user, new_name) - - defer.returnValue((200, {})) - - def on_OPTIONS(self, request, user_id): - return (200, {}) - - -class ProfileAvatarURLRestServlet(RestServlet): - PATTERN = client_path_pattern("/profile/(?P[^/]*)/avatar_url") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) - - avatar_url = yield self.handlers.profile_handler.get_avatar_url( - user, - ) - - defer.returnValue((200, {"avatar_url": avatar_url})) - - @defer.inlineCallbacks - def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - try: - content = json.loads(request.content.read()) - new_name = content["avatar_url"] - except: - defer.returnValue((400, "Unable to parse name")) - - yield self.handlers.profile_handler.set_avatar_url( - user, auth_user, new_name) - - defer.returnValue((200, {})) - - def on_OPTIONS(self, request, user_id): - return (200, {}) - - -class ProfileRestServlet(RestServlet): - PATTERN = client_path_pattern("/profile/(?P[^/]*)") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) - - displayname = yield self.handlers.profile_handler.get_displayname( - user, - ) - avatar_url = yield self.handlers.profile_handler.get_avatar_url( - user, - ) - - defer.returnValue((200, { - "displayname": displayname, - "avatar_url": avatar_url - })) - - -def register_servlets(hs, http_server): - ProfileDisplaynameRestServlet(hs).register(http_server) - ProfileAvatarURLRestServlet(hs).register(http_server) - ProfileRestServlet(hs).register(http_server) diff --git a/synapse/rest/register.py b/synapse/rest/register.py deleted file mode 100644 index e3b26902d9..0000000000 --- a/synapse/rest/register.py +++ /dev/null @@ -1,291 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module contains REST servlets to do with registration: /register""" -from twisted.internet import defer - -from synapse.api.errors import SynapseError, Codes -from synapse.api.constants import LoginType -from base import RestServlet, client_path_pattern -import synapse.util.stringutils as stringutils - -from synapse.util.async import run_on_reactor - -from hashlib import sha1 -import hmac -import json -import logging -import urllib - -logger = logging.getLogger(__name__) - - -# We ought to be using hmac.compare_digest() but on older pythons it doesn't -# exist. It's a _really minor_ security flaw to use plain string comparison -# because the timing attack is so obscured by all the other code here it's -# unlikely to make much difference -if hasattr(hmac, "compare_digest"): - compare_digest = hmac.compare_digest -else: - compare_digest = lambda a, b: a == b - - -class RegisterRestServlet(RestServlet): - """Handles registration with the home server. - - This servlet is in control of the registration flow; the registration - handler doesn't have a concept of multi-stages or sessions. - """ - - PATTERN = client_path_pattern("/register$") - - def __init__(self, hs): - super(RegisterRestServlet, self).__init__(hs) - # sessions are stored as: - # self.sessions = { - # "session_id" : { __session_dict__ } - # } - # TODO: persistent storage - self.sessions = {} - - def on_GET(self, request): - if self.hs.config.enable_registration_captcha: - return ( - 200, - {"flows": [ - { - "type": LoginType.RECAPTCHA, - "stages": [ - LoginType.RECAPTCHA, - LoginType.EMAIL_IDENTITY, - LoginType.PASSWORD - ] - }, - { - "type": LoginType.RECAPTCHA, - "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD] - } - ]} - ) - else: - return ( - 200, - {"flows": [ - { - "type": LoginType.EMAIL_IDENTITY, - "stages": [ - LoginType.EMAIL_IDENTITY, LoginType.PASSWORD - ] - }, - { - "type": LoginType.PASSWORD - } - ]} - ) - - @defer.inlineCallbacks - def on_POST(self, request): - register_json = _parse_json(request) - - session = (register_json["session"] - if "session" in register_json else None) - login_type = None - if "type" not in register_json: - raise SynapseError(400, "Missing 'type' key.") - - try: - login_type = register_json["type"] - stages = { - LoginType.RECAPTCHA: self._do_recaptcha, - LoginType.PASSWORD: self._do_password, - LoginType.EMAIL_IDENTITY: self._do_email_identity - } - - session_info = self._get_session_info(request, session) - logger.debug("%s : session info %s request info %s", - login_type, session_info, register_json) - response = yield stages[login_type]( - request, - register_json, - session_info - ) - - if "access_token" not in response: - # isn't a final response - response["session"] = session_info["id"] - - defer.returnValue((200, response)) - except KeyError as e: - logger.exception(e) - raise SynapseError(400, "Missing JSON keys for login type %s." % ( - login_type, - )) - - def on_OPTIONS(self, request): - return (200, {}) - - def _get_session_info(self, request, session_id): - if not session_id: - # create a new session - while session_id is None or session_id in self.sessions: - session_id = stringutils.random_string(24) - self.sessions[session_id] = { - "id": session_id, - LoginType.EMAIL_IDENTITY: False, - LoginType.RECAPTCHA: False - } - - return self.sessions[session_id] - - def _save_session(self, session): - # TODO: Persistent storage - logger.debug("Saving session %s", session) - self.sessions[session["id"]] = session - - def _remove_session(self, session): - logger.debug("Removing session %s", session) - self.sessions.pop(session["id"]) - - @defer.inlineCallbacks - def _do_recaptcha(self, request, register_json, session): - if not self.hs.config.enable_registration_captcha: - raise SynapseError(400, "Captcha not required.") - - yield self._check_recaptcha(request, register_json, session) - - session[LoginType.RECAPTCHA] = True # mark captcha as done - self._save_session(session) - defer.returnValue({ - "next": [LoginType.PASSWORD, LoginType.EMAIL_IDENTITY] - }) - - @defer.inlineCallbacks - def _check_recaptcha(self, request, register_json, session): - if ("captcha_bypass_hmac" in register_json and - self.hs.config.captcha_bypass_secret): - if "user" not in register_json: - raise SynapseError(400, "Captcha bypass needs 'user'") - - want = hmac.new( - key=self.hs.config.captcha_bypass_secret, - msg=register_json["user"], - digestmod=sha1, - ).hexdigest() - - # str() because otherwise hmac complains that 'unicode' does not - # have the buffer interface - got = str(register_json["captcha_bypass_hmac"]) - - if compare_digest(want, got): - session["user"] = register_json["user"] - defer.returnValue(None) - else: - raise SynapseError( - 400, "Captcha bypass HMAC incorrect", - errcode=Codes.CAPTCHA_NEEDED - ) - - challenge = None - user_response = None - try: - challenge = register_json["challenge"] - user_response = register_json["response"] - except KeyError: - raise SynapseError(400, "Captcha response is required", - errcode=Codes.CAPTCHA_NEEDED) - - ip_addr = self.hs.get_ip_from_request(request) - - handler = self.handlers.registration_handler - yield handler.check_recaptcha( - ip_addr, - self.hs.config.recaptcha_private_key, - challenge, - user_response - ) - - @defer.inlineCallbacks - def _do_email_identity(self, request, register_json, session): - if (self.hs.config.enable_registration_captcha and - not session[LoginType.RECAPTCHA]): - raise SynapseError(400, "Captcha is required.") - - threepidCreds = register_json['threepidCreds'] - handler = self.handlers.registration_handler - logger.debug("Registering email. threepidcreds: %s" % (threepidCreds)) - yield handler.register_email(threepidCreds) - session["threepidCreds"] = threepidCreds # store creds for next stage - session[LoginType.EMAIL_IDENTITY] = True # mark email as done - self._save_session(session) - defer.returnValue({ - "next": LoginType.PASSWORD - }) - - @defer.inlineCallbacks - def _do_password(self, request, register_json, session): - yield run_on_reactor() - if (self.hs.config.enable_registration_captcha and - not session[LoginType.RECAPTCHA]): - # captcha should've been done by this stage! - raise SynapseError(400, "Captcha is required.") - - if ("user" in session and "user" in register_json and - session["user"] != register_json["user"]): - raise SynapseError( - 400, "Cannot change user ID during registration" - ) - - password = register_json["password"].encode("utf-8") - desired_user_id = (register_json["user"].encode("utf-8") - if "user" in register_json else None) - if (desired_user_id - and urllib.quote(desired_user_id) != desired_user_id): - raise SynapseError( - 400, - "User ID must only contain characters which do not " + - "require URL encoding.") - handler = self.handlers.registration_handler - (user_id, token) = yield handler.register( - localpart=desired_user_id, - password=password - ) - - if session[LoginType.EMAIL_IDENTITY]: - logger.debug("Binding emails %s to %s" % ( - session["threepidCreds"], user_id) - ) - yield handler.bind_emails(user_id, session["threepidCreds"]) - - result = { - "user_id": user_id, - "access_token": token, - "home_server": self.hs.hostname, - } - self._remove_session(session) - defer.returnValue(result) - - -def _parse_json(request): - try: - content = json.loads(request.content.read()) - if type(content) != dict: - raise SynapseError(400, "Content must be a JSON object.") - return content - except ValueError: - raise SynapseError(400, "Content not JSON.") - - -def register_servlets(hs, http_server): - RegisterRestServlet(hs).register(http_server) diff --git a/synapse/rest/room.py b/synapse/rest/room.py deleted file mode 100644 index 48bba2a5f3..0000000000 --- a/synapse/rest/room.py +++ /dev/null @@ -1,559 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" This module contains REST servlets to do with rooms: /rooms/ """ -from twisted.internet import defer - -from base import RestServlet, client_path_pattern -from synapse.api.errors import SynapseError, Codes -from synapse.streams.config import PaginationConfig -from synapse.api.constants import EventTypes, Membership - -import json -import logging -import urllib - - -logger = logging.getLogger(__name__) - - -class RoomCreateRestServlet(RestServlet): - # No PATTERN; we have custom dispatch rules here - - def register(self, http_server): - PATTERN = "/createRoom" - register_txn_path(self, PATTERN, http_server) - # define CORS for all of /rooms in RoomCreateRestServlet for simplicity - http_server.register_path("OPTIONS", - client_path_pattern("/rooms(?:/.*)?$"), - self.on_OPTIONS) - # define CORS for /createRoom[/txnid] - http_server.register_path("OPTIONS", - client_path_pattern("/createRoom(?:/.*)?$"), - self.on_OPTIONS) - - @defer.inlineCallbacks - def on_PUT(self, request, txn_id): - try: - defer.returnValue( - self.txns.get_client_transaction(request, txn_id) - ) - except KeyError: - pass - - response = yield self.on_POST(request) - - self.txns.store_client_transaction(request, txn_id, response) - defer.returnValue(response) - - @defer.inlineCallbacks - def on_POST(self, request): - auth_user = yield self.auth.get_user_by_req(request) - - room_config = self.get_room_config(request) - info = yield self.make_room(room_config, auth_user, None) - room_config.update(info) - defer.returnValue((200, info)) - - @defer.inlineCallbacks - def make_room(self, room_config, auth_user, room_id): - handler = self.handlers.room_creation_handler - info = yield handler.create_room( - user_id=auth_user.to_string(), - room_id=room_id, - config=room_config - ) - defer.returnValue(info) - - def get_room_config(self, request): - try: - user_supplied_config = json.loads(request.content.read()) - if "visibility" not in user_supplied_config: - # default visibility - user_supplied_config["visibility"] = "public" - return user_supplied_config - except (ValueError, TypeError): - raise SynapseError(400, "Body must be JSON.", - errcode=Codes.BAD_JSON) - - def on_OPTIONS(self, request): - return (200, {}) - - -# TODO: Needs unit testing for generic events -class RoomStateEventRestServlet(RestServlet): - def register(self, http_server): - # /room/$roomid/state/$eventtype - no_state_key = "/rooms/(?P[^/]*)/state/(?P[^/]*)$" - - # /room/$roomid/state/$eventtype/$statekey - state_key = ("/rooms/(?P[^/]*)/state/" - "(?P[^/]*)/(?P[^/]*)$") - - http_server.register_path("GET", - client_path_pattern(state_key), - self.on_GET) - http_server.register_path("PUT", - client_path_pattern(state_key), - self.on_PUT) - http_server.register_path("GET", - client_path_pattern(no_state_key), - self.on_GET_no_state_key) - http_server.register_path("PUT", - client_path_pattern(no_state_key), - self.on_PUT_no_state_key) - - def on_GET_no_state_key(self, request, room_id, event_type): - return self.on_GET(request, room_id, event_type, "") - - def on_PUT_no_state_key(self, request, room_id, event_type): - return self.on_PUT(request, room_id, event_type, "") - - @defer.inlineCallbacks - def on_GET(self, request, room_id, event_type, state_key): - user = yield self.auth.get_user_by_req(request) - - msg_handler = self.handlers.message_handler - data = yield msg_handler.get_room_data( - user_id=user.to_string(), - room_id=room_id, - event_type=event_type, - state_key=state_key, - ) - - if not data: - raise SynapseError( - 404, "Event not found.", errcode=Codes.NOT_FOUND - ) - defer.returnValue((200, data.get_dict()["content"])) - - @defer.inlineCallbacks - def on_PUT(self, request, room_id, event_type, state_key): - user = yield self.auth.get_user_by_req(request) - - content = _parse_json(request) - - event_dict = { - "type": event_type, - "content": content, - "room_id": room_id, - "sender": user.to_string(), - } - - if state_key is not None: - event_dict["state_key"] = state_key - - msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event(event_dict) - - defer.returnValue((200, {})) - - -# TODO: Needs unit testing for generic events + feedback -class RoomSendEventRestServlet(RestServlet): - - def register(self, http_server): - # /rooms/$roomid/send/$event_type[/$txn_id] - PATTERN = ("/rooms/(?P[^/]*)/send/(?P[^/]*)") - register_txn_path(self, PATTERN, http_server, with_get=True) - - @defer.inlineCallbacks - def on_POST(self, request, room_id, event_type): - user = yield self.auth.get_user_by_req(request) - content = _parse_json(request) - - msg_handler = self.handlers.message_handler - event = yield msg_handler.create_and_send_event( - { - "type": event_type, - "content": content, - "room_id": room_id, - "sender": user.to_string(), - } - ) - - defer.returnValue((200, {"event_id": event.event_id})) - - def on_GET(self, request, room_id, event_type, txn_id): - return (200, "Not implemented") - - @defer.inlineCallbacks - def on_PUT(self, request, room_id, event_type, txn_id): - try: - defer.returnValue( - self.txns.get_client_transaction(request, txn_id) - ) - except KeyError: - pass - - response = yield self.on_POST(request, room_id, event_type) - - self.txns.store_client_transaction(request, txn_id, response) - defer.returnValue(response) - - -# TODO: Needs unit testing for room ID + alias joins -class JoinRoomAliasServlet(RestServlet): - - def register(self, http_server): - # /join/$room_identifier[/$txn_id] - PATTERN = ("/join/(?P[^/]*)") - register_txn_path(self, PATTERN, http_server) - - @defer.inlineCallbacks - def on_POST(self, request, room_identifier): - user = yield self.auth.get_user_by_req(request) - - # the identifier could be a room alias or a room id. Try one then the - # other if it fails to parse, without swallowing other valid - # SynapseErrors. - - identifier = None - is_room_alias = False - try: - identifier = self.hs.parse_roomalias(room_identifier) - is_room_alias = True - except SynapseError: - identifier = self.hs.parse_roomid(room_identifier) - - # TODO: Support for specifying the home server to join with? - - if is_room_alias: - handler = self.handlers.room_member_handler - ret_dict = yield handler.join_room_alias(user, identifier) - defer.returnValue((200, ret_dict)) - else: # room id - msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event( - { - "type": EventTypes.Member, - "content": {"membership": Membership.JOIN}, - "room_id": identifier.to_string(), - "sender": user.to_string(), - "state_key": user.to_string(), - } - ) - - defer.returnValue((200, {"room_id": identifier.to_string()})) - - @defer.inlineCallbacks - def on_PUT(self, request, room_identifier, txn_id): - try: - defer.returnValue( - self.txns.get_client_transaction(request, txn_id) - ) - except KeyError: - pass - - response = yield self.on_POST(request, room_identifier) - - self.txns.store_client_transaction(request, txn_id, response) - defer.returnValue(response) - - -# TODO: Needs unit testing -class PublicRoomListRestServlet(RestServlet): - PATTERN = client_path_pattern("/publicRooms$") - - @defer.inlineCallbacks - def on_GET(self, request): - handler = self.handlers.room_list_handler - data = yield handler.get_public_room_list() - defer.returnValue((200, data)) - - -# TODO: Needs unit testing -class RoomMemberListRestServlet(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/members$") - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - # TODO support Pagination stream API (limit/tokens) - user = yield self.auth.get_user_by_req(request) - handler = self.handlers.room_member_handler - members = yield handler.get_room_members_as_pagination_chunk( - room_id=room_id, - user_id=user.to_string()) - - for event in members["chunk"]: - # FIXME: should probably be state_key here, not user_id - target_user = self.hs.parse_userid(event["user_id"]) - # Presence is an optional cache; don't fail if we can't fetch it - try: - presence_handler = self.handlers.presence_handler - presence_state = yield presence_handler.get_state( - target_user=target_user, auth_user=user - ) - event["content"].update(presence_state) - except: - pass - - defer.returnValue((200, members)) - - -# TODO: Needs unit testing -class RoomMessageListRestServlet(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/messages$") - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) - pagination_config = PaginationConfig.from_request( - request, default_limit=10, - ) - with_feedback = "feedback" in request.args - as_client_event = "raw" not in request.args - handler = self.handlers.message_handler - msgs = yield handler.get_messages( - room_id=room_id, - user_id=user.to_string(), - pagin_config=pagination_config, - feedback=with_feedback, - as_client_event=as_client_event - ) - - defer.returnValue((200, msgs)) - - -# TODO: Needs unit testing -class RoomStateRestServlet(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/state$") - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) - handler = self.handlers.message_handler - # Get all the current state for this room - events = yield handler.get_state_events( - room_id=room_id, - user_id=user.to_string(), - ) - defer.returnValue((200, events)) - - -# TODO: Needs unit testing -class RoomInitialSyncRestServlet(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/initialSync$") - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) - pagination_config = PaginationConfig.from_request(request) - content = yield self.handlers.message_handler.room_initial_sync( - room_id=room_id, - user_id=user.to_string(), - pagin_config=pagination_config, - ) - defer.returnValue((200, content)) - - -class RoomTriggerBackfill(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/backfill$") - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - remote_server = urllib.unquote( - request.args["remote"][0] - ).decode("UTF-8") - - limit = int(request.args["limit"][0]) - - handler = self.handlers.federation_handler - events = yield handler.backfill(remote_server, room_id, limit) - - res = [self.hs.serialize_event(event) for event in events] - defer.returnValue((200, res)) - - -# TODO: Needs unit testing -class RoomMembershipRestServlet(RestServlet): - - def register(self, http_server): - # /rooms/$roomid/[invite|join|leave] - PATTERN = ("/rooms/(?P[^/]*)/" - "(?Pjoin|invite|leave|ban|kick)") - register_txn_path(self, PATTERN, http_server) - - @defer.inlineCallbacks - def on_POST(self, request, room_id, membership_action): - user = yield self.auth.get_user_by_req(request) - - content = _parse_json(request) - - # target user is you unless it is an invite - state_key = user.to_string() - if membership_action in ["invite", "ban", "kick"]: - if "user_id" not in content: - raise SynapseError(400, "Missing user_id key.") - state_key = content["user_id"] - - if membership_action == "kick": - membership_action = "leave" - - msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event( - { - "type": EventTypes.Member, - "content": {"membership": unicode(membership_action)}, - "room_id": room_id, - "sender": user.to_string(), - "state_key": state_key, - } - ) - - defer.returnValue((200, {})) - - @defer.inlineCallbacks - def on_PUT(self, request, room_id, membership_action, txn_id): - try: - defer.returnValue( - self.txns.get_client_transaction(request, txn_id) - ) - except KeyError: - pass - - response = yield self.on_POST(request, room_id, membership_action) - - self.txns.store_client_transaction(request, txn_id, response) - defer.returnValue(response) - - -class RoomRedactEventRestServlet(RestServlet): - def register(self, http_server): - PATTERN = ("/rooms/(?P[^/]*)/redact/(?P[^/]*)") - register_txn_path(self, PATTERN, http_server) - - @defer.inlineCallbacks - def on_POST(self, request, room_id, event_id): - user = yield self.auth.get_user_by_req(request) - content = _parse_json(request) - - msg_handler = self.handlers.message_handler - event = yield msg_handler.create_and_send_event( - { - "type": EventTypes.Redaction, - "content": content, - "room_id": room_id, - "sender": user.to_string(), - "redacts": event_id, - } - ) - - defer.returnValue((200, {"event_id": event.event_id})) - - @defer.inlineCallbacks - def on_PUT(self, request, room_id, event_id, txn_id): - try: - defer.returnValue( - self.txns.get_client_transaction(request, txn_id) - ) - except KeyError: - pass - - response = yield self.on_POST(request, room_id, event_id) - - self.txns.store_client_transaction(request, txn_id, response) - defer.returnValue(response) - - -class RoomTypingRestServlet(RestServlet): - PATTERN = client_path_pattern( - "/rooms/(?P[^/]*)/typing/(?P[^/]*)$" - ) - - @defer.inlineCallbacks - def on_PUT(self, request, room_id, user_id): - auth_user = yield self.auth.get_user_by_req(request) - - room_id = urllib.unquote(room_id) - target_user = self.hs.parse_userid(urllib.unquote(user_id)) - - content = _parse_json(request) - - typing_handler = self.handlers.typing_notification_handler - - if content["typing"]: - yield typing_handler.started_typing( - target_user=target_user, - auth_user=auth_user, - room_id=room_id, - timeout=content.get("timeout", 30000), - ) - else: - yield typing_handler.stopped_typing( - target_user=target_user, - auth_user=auth_user, - room_id=room_id, - ) - - defer.returnValue((200, {})) - - -def _parse_json(request): - try: - content = json.loads(request.content.read()) - if type(content) != dict: - raise SynapseError(400, "Content must be a JSON object.", - errcode=Codes.NOT_JSON) - return content - except ValueError: - raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) - - -def register_txn_path(servlet, regex_string, http_server, with_get=False): - """Registers a transaction-based path. - - This registers two paths: - PUT regex_string/$txnid - POST regex_string - - Args: - regex_string (str): The regex string to register. Must NOT have a - trailing $ as this string will be appended to. - http_server : The http_server to register paths with. - with_get: True to also register respective GET paths for the PUTs. - """ - http_server.register_path( - "POST", - client_path_pattern(regex_string + "$"), - servlet.on_POST - ) - http_server.register_path( - "PUT", - client_path_pattern(regex_string + "/(?P[^/]*)$"), - servlet.on_PUT - ) - if with_get: - http_server.register_path( - "GET", - client_path_pattern(regex_string + "/(?P[^/]*)$"), - servlet.on_GET - ) - - -def register_servlets(hs, http_server): - RoomStateEventRestServlet(hs).register(http_server) - RoomCreateRestServlet(hs).register(http_server) - RoomMemberListRestServlet(hs).register(http_server) - RoomMessageListRestServlet(hs).register(http_server) - JoinRoomAliasServlet(hs).register(http_server) - RoomTriggerBackfill(hs).register(http_server) - RoomMembershipRestServlet(hs).register(http_server) - RoomSendEventRestServlet(hs).register(http_server) - PublicRoomListRestServlet(hs).register(http_server) - RoomStateRestServlet(hs).register(http_server) - RoomInitialSyncRestServlet(hs).register(http_server) - RoomRedactEventRestServlet(hs).register(http_server) - RoomTypingRestServlet(hs).register(http_server) diff --git a/synapse/rest/transactions.py b/synapse/rest/transactions.py deleted file mode 100644 index d933fea18a..0000000000 --- a/synapse/rest/transactions.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module contains logic for storing HTTP PUT transactions. This is used -to ensure idempotency when performing PUTs using the REST API.""" -import logging - -logger = logging.getLogger(__name__) - - -# FIXME: elsewhere we use FooStore to indicate something in the storage layer... -class HttpTransactionStore(object): - - def __init__(self): - # { key : (txn_id, response) } - self.transactions = {} - - def get_response(self, key, txn_id): - """Retrieve a response for this request. - - Args: - key (str): A transaction-independent key for this request. Usually - this is a combination of the path (without the transaction id) - and the user's access token. - txn_id (str): The transaction ID for this request - Returns: - A tuple of (HTTP response code, response content) or None. - """ - try: - logger.debug("get_response Key: %s TxnId: %s", key, txn_id) - (last_txn_id, response) = self.transactions[key] - if txn_id == last_txn_id: - logger.info("get_response: Returning a response for %s", key) - return response - except KeyError: - pass - return None - - def store_response(self, key, txn_id, response): - """Stores an HTTP response tuple. - - Args: - key (str): A transaction-independent key for this request. Usually - this is a combination of the path (without the transaction id) - and the user's access token. - txn_id (str): The transaction ID for this request. - response (tuple): A tuple of (HTTP response code, response content) - """ - logger.debug("store_response Key: %s TxnId: %s", key, txn_id) - self.transactions[key] = (txn_id, response) - - def store_client_transaction(self, request, txn_id, response): - """Stores the request/response pair of an HTTP transaction. - - Args: - request (twisted.web.http.Request): The twisted HTTP request. This - request must have the transaction ID as the last path segment. - response (tuple): A tuple of (response code, response dict) - txn_id (str): The transaction ID for this request. - """ - self.store_response(self._get_key(request), txn_id, response) - - def get_client_transaction(self, request, txn_id): - """Retrieves a stored response if there was one. - - Args: - request (twisted.web.http.Request): The twisted HTTP request. This - request must have the transaction ID as the last path segment. - txn_id (str): The transaction ID for this request. - Returns: - The response tuple. - Raises: - KeyError if the transaction was not found. - """ - response = self.get_response(self._get_key(request), txn_id) - if response is None: - raise KeyError("Transaction not found.") - return response - - def _get_key(self, request): - token = request.args["access_token"][0] - path_without_txn_id = request.path.rsplit("/", 1)[0] - return path_without_txn_id + "/" + token diff --git a/synapse/rest/voip.py b/synapse/rest/voip.py deleted file mode 100644 index 011c35e69b..0000000000 --- a/synapse/rest/voip.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from base import RestServlet, client_path_pattern - - -import hmac -import hashlib -import base64 - - -class VoipRestServlet(RestServlet): - PATTERN = client_path_pattern("/voip/turnServer$") - - @defer.inlineCallbacks - def on_GET(self, request): - auth_user = yield self.auth.get_user_by_req(request) - - turnUris = self.hs.config.turn_uris - turnSecret = self.hs.config.turn_shared_secret - userLifetime = self.hs.config.turn_user_lifetime - if not turnUris or not turnSecret or not userLifetime: - defer.returnValue((200, {})) - - expiry = (self.hs.get_clock().time_msec() + userLifetime) / 1000 - username = "%d:%s" % (expiry, auth_user.to_string()) - - mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1) - # We need to use standard base64 encoding here, *not* syutil's - # encode_base64 because we need to add the standard padding to get the - # same result as the TURN server. - password = base64.b64encode(mac.digest()) - - defer.returnValue((200, { - 'username': username, - 'password': password, - 'ttl': userLifetime / 1000, - 'uris': turnUris, - })) - - def on_OPTIONS(self, request): - return (200, {}) - - -def register_servlets(hs, http_server): - VoipRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index d861efd2fd..57a95bf753 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -24,7 +24,7 @@ from synapse.events.utils import serialize_event from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers -from synapse.rest import RestServletFactory +from synapse.client.v1 import RestServletFactory from synapse.state import StateHandler from synapse.storage import DataStore from synapse.types import UserID, RoomAlias, RoomID, EventID diff --git a/tests/client/__init__.py b/tests/client/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/tests/client/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/client/v1/__init__.py b/tests/client/v1/__init__.py new file mode 100644 index 0000000000..9bff9ec169 --- /dev/null +++ b/tests/client/v1/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/client/v1/test_events.py b/tests/client/v1/test_events.py new file mode 100644 index 0000000000..9b36dd3225 --- /dev/null +++ b/tests/client/v1/test_events.py @@ -0,0 +1,225 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests REST events for /events paths.""" +from tests import unittest + +# twisted imports +from twisted.internet import defer + +import synapse.client.v1.events +import synapse.client.v1.register +import synapse.client.v1.room + +from synapse.server import HomeServer + +from ...utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from .utils import RestTestCase + +from mock import Mock, NonCallableMock + + +PATH_PREFIX = "/_matrix/client/api/v1" + + +class EventStreamPaginationApiTestCase(unittest.TestCase): + """ Tests event streaming query parameters and start/end keys used in the + Pagination stream API. """ + user_id = "sid1" + + def setUp(self): + # configure stream and inject items + pass + + def tearDown(self): + pass + + def TODO_test_long_poll(self): + # stream from 'end' key, send (self+other) message, expect message. + + # stream from 'END', send (self+other) message, expect message. + + # stream from 'end' key, send (self+other) topic, expect topic. + + # stream from 'END', send (self+other) topic, expect topic. + + # stream from 'end' key, send (self+other) invite, expect invite. + + # stream from 'END', send (self+other) invite, expect invite. + + pass + + def TODO_test_stream_forward(self): + # stream from START, expect injected items + + # stream from 'start' key, expect same content + + # stream from 'end' key, expect nothing + + # stream from 'END', expect nothing + + # The following is needed for cases where content is removed e.g. you + # left a room, so the token you're streaming from is > the one that + # would be returned naturally from START>END. + # stream from very new token (higher than end key), expect same token + # returned as end key + pass + + def TODO_test_limits(self): + # stream from a key, expect limit_num items + + # stream from START, expect limit_num items + + pass + + def TODO_test_range(self): + # stream from key to key, expect X items + + # stream from key to END, expect X items + + # stream from START to key, expect X items + + # stream from START to END, expect all items + pass + + def TODO_test_direction(self): + # stream from END to START and fwds, expect newest first + + # stream from END to START and bwds, expect oldest first + + # stream from START to END and fwds, expect oldest first + + # stream from START to END and bwds, expect newest first + + pass + + +class EventStreamPermissionsTestCase(RestTestCase): + """ Tests event streaming (GET /events). """ + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "test", + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + clock=Mock(spec=[ + "call_later", + "cancel_call_later", + "time_msec", + "time" + ]), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + hs.config.enable_registration_captcha = False + + hs.get_handlers().federation_handler = Mock() + + hs.get_clock().time_msec.return_value = 1000000 + hs.get_clock().time.return_value = 1000 + + synapse.client.v1.register.register_servlets(hs, self.mock_resource) + synapse.client.v1.events.register_servlets(hs, self.mock_resource) + synapse.client.v1.room.register_servlets(hs, self.mock_resource) + + # register an account + self.user_id = "sid1" + response = yield self.register(self.user_id) + self.token = response["access_token"] + self.user_id = response["user_id"] + + # register a 2nd account + self.other_user = "other1" + response = yield self.register(self.other_user) + self.other_token = response["access_token"] + self.other_user = response["user_id"] + + def tearDown(self): + pass + + @defer.inlineCallbacks + def test_stream_basic_permissions(self): + # invalid token, expect 403 + (code, response) = yield self.mock_resource.trigger_get( + "/events?access_token=%s" % ("invalid" + self.token, ) + ) + self.assertEquals(403, code, msg=str(response)) + + # valid token, expect content + (code, response) = yield self.mock_resource.trigger_get( + "/events?access_token=%s&timeout=0" % (self.token,) + ) + self.assertEquals(200, code, msg=str(response)) + self.assertTrue("chunk" in response) + self.assertTrue("start" in response) + self.assertTrue("end" in response) + + @defer.inlineCallbacks + def test_stream_room_permissions(self): + room_id = yield self.create_room_as( + self.other_user, + tok=self.other_token + ) + yield self.send(room_id, tok=self.other_token) + + # invited to room (expect no content for room) + yield self.invite( + room_id, + src=self.other_user, + targ=self.user_id, + tok=self.other_token + ) + + (code, response) = yield self.mock_resource.trigger_get( + "/events?access_token=%s&timeout=0" % (self.token,) + ) + self.assertEquals(200, code, msg=str(response)) + + self.assertEquals(0, len(response["chunk"])) + + # joined room (expect all content for room) + yield self.join(room=room_id, user=self.user_id, tok=self.token) + + # left to room (expect no content for room) + + def TODO_test_stream_items(self): + # new user, no content + + # join room, expect 1 item (join) + + # send message, expect 2 items (join,send) + + # set topic, expect 3 items (join,send,topic) + + # someone else join room, expect 4 (join,send,topic,join) + + # someone else send message, expect 5 (join,send.topic,join,send) + + # someone else set topic, expect 6 (join,send,topic,join,send,topic) + pass diff --git a/tests/client/v1/test_presence.py b/tests/client/v1/test_presence.py new file mode 100644 index 0000000000..e7d636c74d --- /dev/null +++ b/tests/client/v1/test_presence.py @@ -0,0 +1,372 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests REST events for /presence paths.""" + +from tests import unittest +from twisted.internet import defer + +from mock import Mock + +from ...utils import MockHttpResource, MockKey + +from synapse.api.constants import PresenceState +from synapse.handlers.presence import PresenceHandler +from synapse.server import HomeServer + + +OFFLINE = PresenceState.OFFLINE +UNAVAILABLE = PresenceState.UNAVAILABLE +ONLINE = PresenceState.ONLINE + + +myid = "@apple:test" +PATH_PREFIX = "/_matrix/client/api/v1" + + +class JustPresenceHandlers(object): + def __init__(self, hs): + self.presence_handler = PresenceHandler(hs) + + +class PresenceStateTestCase(unittest.TestCase): + + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", + db_pool=None, + datastore=Mock(spec=[ + "get_presence_state", + "set_presence_state", + "insert_client_ip", + ]), + http_client=None, + resource_for_client=self.mock_resource, + resource_for_federation=self.mock_resource, + config=self.mock_config, + ) + hs.handlers = JustPresenceHandlers(hs) + + self.datastore = hs.get_datastore() + + def get_presence_list(*a, **kw): + return defer.succeed([]) + self.datastore.get_presence_list = get_presence_list + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(myid), + "admin": False, + "device_id": None, + } + + hs.get_auth().get_user_by_token = _get_user_by_token + + room_member_handler = hs.handlers.room_member_handler = Mock( + spec=[ + "get_rooms_for_user", + ] + ) + + def get_rooms_for_user(user): + return defer.succeed([]) + room_member_handler.get_rooms_for_user = get_rooms_for_user + + hs.register_servlets() + + self.u_apple = hs.parse_userid(myid) + + @defer.inlineCallbacks + def test_get_my_status(self): + mocked_get = self.datastore.get_presence_state + mocked_get.return_value = defer.succeed( + {"state": ONLINE, "status_msg": "Available"} + ) + + (code, response) = yield self.mock_resource.trigger("GET", + "/presence/%s/status" % (myid), None) + + self.assertEquals(200, code) + self.assertEquals( + {"presence": ONLINE, "status_msg": "Available"}, + response + ) + mocked_get.assert_called_with("apple") + + @defer.inlineCallbacks + def test_set_my_status(self): + mocked_set = self.datastore.set_presence_state + mocked_set.return_value = defer.succeed({"state": OFFLINE}) + + (code, response) = yield self.mock_resource.trigger("PUT", + "/presence/%s/status" % (myid), + '{"presence": "unavailable", "status_msg": "Away"}') + + self.assertEquals(200, code) + mocked_set.assert_called_with("apple", + {"state": UNAVAILABLE, "status_msg": "Away"} + ) + + +class PresenceListTestCase(unittest.TestCase): + + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + + hs = HomeServer("test", + db_pool=None, + datastore=Mock(spec=[ + "has_presence_state", + "get_presence_state", + "allow_presence_visible", + "is_presence_visible", + "add_presence_list_pending", + "set_presence_list_accepted", + "del_presence_list", + "get_presence_list", + "insert_client_ip", + ]), + http_client=None, + resource_for_client=self.mock_resource, + resource_for_federation=self.mock_resource, + config=self.mock_config, + ) + hs.handlers = JustPresenceHandlers(hs) + + self.datastore = hs.get_datastore() + + def has_presence_state(user_localpart): + return defer.succeed( + user_localpart in ("apple", "banana",) + ) + self.datastore.has_presence_state = has_presence_state + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(myid), + "admin": False, + "device_id": None, + } + + room_member_handler = hs.handlers.room_member_handler = Mock( + spec=[ + "get_rooms_for_user", + ] + ) + + hs.get_auth().get_user_by_token = _get_user_by_token + + hs.register_servlets() + + self.u_apple = hs.parse_userid("@apple:test") + self.u_banana = hs.parse_userid("@banana:test") + + @defer.inlineCallbacks + def test_get_my_list(self): + self.datastore.get_presence_list.return_value = defer.succeed( + [{"observed_user_id": "@banana:test"}], + ) + + (code, response) = yield self.mock_resource.trigger("GET", + "/presence/list/%s" % (myid), None) + + self.assertEquals(200, code) + self.assertEquals([ + {"user_id": "@banana:test", "presence": OFFLINE}, + ], response) + + self.datastore.get_presence_list.assert_called_with( + "apple", accepted=True + ) + + @defer.inlineCallbacks + def test_invite(self): + self.datastore.add_presence_list_pending.return_value = ( + defer.succeed(()) + ) + self.datastore.is_presence_visible.return_value = defer.succeed( + True + ) + + (code, response) = yield self.mock_resource.trigger("POST", + "/presence/list/%s" % (myid), + """{"invite": ["@banana:test"]}""" + ) + + self.assertEquals(200, code) + + self.datastore.add_presence_list_pending.assert_called_with( + "apple", "@banana:test" + ) + self.datastore.set_presence_list_accepted.assert_called_with( + "apple", "@banana:test" + ) + + @defer.inlineCallbacks + def test_drop(self): + self.datastore.del_presence_list.return_value = ( + defer.succeed(()) + ) + + (code, response) = yield self.mock_resource.trigger("POST", + "/presence/list/%s" % (myid), + """{"drop": ["@banana:test"]}""" + ) + + self.assertEquals(200, code) + + self.datastore.del_presence_list.assert_called_with( + "apple", "@banana:test" + ) + + +class PresenceEventStreamTestCase(unittest.TestCase): + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + + # HIDEOUS HACKERY + # TODO(paul): This should be injected in via the HomeServer DI system + from synapse.streams.events import ( + PresenceEventSource, NullSource, EventSources + ) + + old_SOURCE_TYPES = EventSources.SOURCE_TYPES + def tearDown(): + EventSources.SOURCE_TYPES = old_SOURCE_TYPES + self.tearDown = tearDown + + EventSources.SOURCE_TYPES = { + k: NullSource for k in old_SOURCE_TYPES.keys() + } + EventSources.SOURCE_TYPES["presence"] = PresenceEventSource + + hs = HomeServer("test", + db_pool=None, + http_client=None, + resource_for_client=self.mock_resource, + resource_for_federation=self.mock_resource, + datastore=Mock(spec=[ + "set_presence_state", + "get_presence_list", + ]), + clock=Mock(spec=[ + "call_later", + "cancel_call_later", + "time_msec", + ]), + config=self.mock_config, + ) + + hs.get_clock().time_msec.return_value = 1000000 + + def _get_user_by_req(req=None): + return hs.parse_userid(myid) + + hs.get_auth().get_user_by_req = _get_user_by_req + + hs.register_servlets() + + hs.handlers.room_member_handler = Mock(spec=[]) + + self.room_members = [] + + def get_rooms_for_user(user): + if user in self.room_members: + return ["a-room"] + else: + return [] + hs.handlers.room_member_handler.get_rooms_for_user = get_rooms_for_user + + self.mock_datastore = hs.get_datastore() + + def get_profile_displayname(user_id): + return defer.succeed("Frank") + self.mock_datastore.get_profile_displayname = get_profile_displayname + + def get_profile_avatar_url(user_id): + return defer.succeed(None) + self.mock_datastore.get_profile_avatar_url = get_profile_avatar_url + + def user_rooms_intersect(user_list): + room_member_ids = map(lambda u: u.to_string(), self.room_members) + + shared = all(map(lambda i: i in room_member_ids, user_list)) + return defer.succeed(shared) + self.mock_datastore.user_rooms_intersect = user_rooms_intersect + + def get_joined_hosts_for_room(room_id): + return [] + self.mock_datastore.get_joined_hosts_for_room = get_joined_hosts_for_room + + self.presence = hs.get_handlers().presence_handler + + self.u_apple = hs.parse_userid("@apple:test") + self.u_banana = hs.parse_userid("@banana:test") + + @defer.inlineCallbacks + def test_shortpoll(self): + self.room_members = [self.u_apple, self.u_banana] + + self.mock_datastore.set_presence_state.return_value = defer.succeed( + {"state": ONLINE} + ) + self.mock_datastore.get_presence_list.return_value = defer.succeed( + [] + ) + + (code, response) = yield self.mock_resource.trigger("GET", + "/events?timeout=0", None) + + self.assertEquals(200, code) + + # We've forced there to be only one data stream so the tokens will + # all be ours + + # I'll already get my own presence state change + self.assertEquals({"start": "0_1_0", "end": "0_1_0", "chunk": []}, + response + ) + + self.mock_datastore.set_presence_state.return_value = defer.succeed( + {"state": ONLINE} + ) + self.mock_datastore.get_presence_list.return_value = defer.succeed( + [] + ) + + yield self.presence.set_state(self.u_banana, self.u_banana, + state={"presence": ONLINE} + ) + + (code, response) = yield self.mock_resource.trigger("GET", + "/events?from=0_1_0&timeout=0", None) + + self.assertEquals(200, code) + self.assertEquals({"start": "0_1_0", "end": "0_2_0", "chunk": [ + {"type": "m.presence", + "content": { + "user_id": "@banana:test", + "presence": ONLINE, + "displayname": "Frank", + "last_active_ago": 0, + }}, + ]}, response) diff --git a/tests/client/v1/test_profile.py b/tests/client/v1/test_profile.py new file mode 100644 index 0000000000..1182cc54eb --- /dev/null +++ b/tests/client/v1/test_profile.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests REST events for /profile paths.""" + +from tests import unittest +from twisted.internet import defer + +from mock import Mock, NonCallableMock + +from ...utils import MockHttpResource, MockKey + +from synapse.api.errors import SynapseError, AuthError +from synapse.server import HomeServer + +myid = "@1234ABCD:test" +PATH_PREFIX = "/_matrix/client/api/v1" + + +class ProfileTestCase(unittest.TestCase): + """ Tests profile management. """ + + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.mock_handler = Mock(spec=[ + "get_displayname", + "set_displayname", + "get_avatar_url", + "set_avatar_url", + ]) + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + hs = HomeServer("test", + db_pool=None, + http_client=None, + resource_for_client=self.mock_resource, + federation=Mock(), + replication_layer=Mock(), + datastore=None, + config=self.mock_config, + ) + + def _get_user_by_req(request=None): + return hs.parse_userid(myid) + + hs.get_auth().get_user_by_req = _get_user_by_req + + hs.get_handlers().profile_handler = self.mock_handler + + hs.register_servlets() + + @defer.inlineCallbacks + def test_get_my_name(self): + mocked_get = self.mock_handler.get_displayname + mocked_get.return_value = defer.succeed("Frank") + + (code, response) = yield self.mock_resource.trigger("GET", + "/profile/%s/displayname" % (myid), None) + + self.assertEquals(200, code) + self.assertEquals({"displayname": "Frank"}, response) + self.assertEquals(mocked_get.call_args[0][0].localpart, "1234ABCD") + + @defer.inlineCallbacks + def test_set_my_name(self): + mocked_set = self.mock_handler.set_displayname + mocked_set.return_value = defer.succeed(()) + + (code, response) = yield self.mock_resource.trigger("PUT", + "/profile/%s/displayname" % (myid), + '{"displayname": "Frank Jr."}') + + self.assertEquals(200, code) + self.assertEquals(mocked_set.call_args[0][0].localpart, "1234ABCD") + self.assertEquals(mocked_set.call_args[0][1].localpart, "1234ABCD") + self.assertEquals(mocked_set.call_args[0][2], "Frank Jr.") + + @defer.inlineCallbacks + def test_set_my_name_noauth(self): + mocked_set = self.mock_handler.set_displayname + mocked_set.side_effect = AuthError(400, "message") + + (code, response) = yield self.mock_resource.trigger("PUT", + "/profile/%s/displayname" % ("@4567:test"), '"Frank Jr."') + + self.assertTrue(400 <= code < 499, + msg="code %d is in the 4xx range" % (code)) + + @defer.inlineCallbacks + def test_get_other_name(self): + mocked_get = self.mock_handler.get_displayname + mocked_get.return_value = defer.succeed("Bob") + + (code, response) = yield self.mock_resource.trigger("GET", + "/profile/%s/displayname" % ("@opaque:elsewhere"), None) + + self.assertEquals(200, code) + self.assertEquals({"displayname": "Bob"}, response) + + @defer.inlineCallbacks + def test_set_other_name(self): + mocked_set = self.mock_handler.set_displayname + mocked_set.side_effect = SynapseError(400, "message") + + (code, response) = yield self.mock_resource.trigger("PUT", + "/profile/%s/displayname" % ("@opaque:elsewhere"), None) + + self.assertTrue(400 <= code <= 499, + msg="code %d is in the 4xx range" % (code)) + + @defer.inlineCallbacks + def test_get_my_avatar(self): + mocked_get = self.mock_handler.get_avatar_url + mocked_get.return_value = defer.succeed("http://my.server/me.png") + + (code, response) = yield self.mock_resource.trigger("GET", + "/profile/%s/avatar_url" % (myid), None) + + self.assertEquals(200, code) + self.assertEquals({"avatar_url": "http://my.server/me.png"}, response) + self.assertEquals(mocked_get.call_args[0][0].localpart, "1234ABCD") + + @defer.inlineCallbacks + def test_set_my_avatar(self): + mocked_set = self.mock_handler.set_avatar_url + mocked_set.return_value = defer.succeed(()) + + (code, response) = yield self.mock_resource.trigger("PUT", + "/profile/%s/avatar_url" % (myid), + '{"avatar_url": "http://my.server/pic.gif"}') + + self.assertEquals(200, code) + self.assertEquals(mocked_set.call_args[0][0].localpart, "1234ABCD") + self.assertEquals(mocked_set.call_args[0][1].localpart, "1234ABCD") + self.assertEquals(mocked_set.call_args[0][2], + "http://my.server/pic.gif") diff --git a/tests/client/v1/test_rooms.py b/tests/client/v1/test_rooms.py new file mode 100644 index 0000000000..33a8631d76 --- /dev/null +++ b/tests/client/v1/test_rooms.py @@ -0,0 +1,1068 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests REST events for /rooms paths.""" + +# twisted imports +from twisted.internet import defer + +import synapse.client.v1.room +from synapse.api.constants import Membership + +from synapse.server import HomeServer + +from tests import unittest + +# python imports +import json +import urllib +import types + +from ...utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from .utils import RestTestCase + +from mock import Mock, NonCallableMock + +PATH_PREFIX = "/_matrix/client/api/v1" + + +class RoomPermissionsTestCase(RestTestCase): + """ Tests room permissions. """ + user_id = "@sid1:red" + rmcreator_id = "@notme:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "red", + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + + self.auth_user_id = self.rmcreator_id + + synapse.client.v1.room.register_servlets(hs, self.mock_resource) + + self.auth = hs.get_auth() + + # create some rooms under the name rmcreator_id + self.uncreated_rmid = "!aa:test" + + self.created_rmid = yield self.create_room_as(self.rmcreator_id, + is_public=False) + + self.created_public_rmid = yield self.create_room_as(self.rmcreator_id, + is_public=True) + + # send a message in one of the rooms + self.created_rmid_msg_path = ("/rooms/%s/send/m.room.message/a1" % + (self.created_rmid)) + (code, response) = yield self.mock_resource.trigger( + "PUT", + self.created_rmid_msg_path, + '{"msgtype":"m.text","body":"test msg"}') + self.assertEquals(200, code, msg=str(response)) + + # set topic for public room + (code, response) = yield self.mock_resource.trigger( + "PUT", + "/rooms/%s/state/m.room.topic" % self.created_public_rmid, + '{"topic":"Public Room Topic"}') + self.assertEquals(200, code, msg=str(response)) + + # auth as user_id now + self.auth_user_id = self.user_id + + def tearDown(self): + pass + +# @defer.inlineCallbacks +# def test_get_message(self): +# # get message in uncreated room, expect 403 +# (code, response) = yield self.mock_resource.trigger_get( +# "/rooms/noroom/messages/someid/m1") +# self.assertEquals(403, code, msg=str(response)) +# +# # get message in created room not joined (no state), expect 403 +# (code, response) = yield self.mock_resource.trigger_get( +# self.created_rmid_msg_path) +# self.assertEquals(403, code, msg=str(response)) +# +# # get message in created room and invited, expect 403 +# yield self.invite(room=self.created_rmid, src=self.rmcreator_id, +# targ=self.user_id) +# (code, response) = yield self.mock_resource.trigger_get( +# self.created_rmid_msg_path) +# self.assertEquals(403, code, msg=str(response)) +# +# # get message in created room and joined, expect 200 +# yield self.join(room=self.created_rmid, user=self.user_id) +# (code, response) = yield self.mock_resource.trigger_get( +# self.created_rmid_msg_path) +# self.assertEquals(200, code, msg=str(response)) +# +# # get message in created room and left, expect 403 +# yield self.leave(room=self.created_rmid, user=self.user_id) +# (code, response) = yield self.mock_resource.trigger_get( +# self.created_rmid_msg_path) +# self.assertEquals(403, code, msg=str(response)) + + @defer.inlineCallbacks + def test_send_message(self): + msg_content = '{"msgtype":"m.text","body":"hello"}' + send_msg_path = ( + "/rooms/%s/send/m.room.message/mid1" % (self.created_rmid,) + ) + + # send message in uncreated room, expect 403 + (code, response) = yield self.mock_resource.trigger( + "PUT", + "/rooms/%s/send/m.room.message/mid2" % (self.uncreated_rmid,), + msg_content + ) + self.assertEquals(403, code, msg=str(response)) + + # send message in created room not joined (no state), expect 403 + (code, response) = yield self.mock_resource.trigger( + "PUT", + send_msg_path, + msg_content + ) + self.assertEquals(403, code, msg=str(response)) + + # send message in created room and invited, expect 403 + yield self.invite( + room=self.created_rmid, + src=self.rmcreator_id, + targ=self.user_id + ) + (code, response) = yield self.mock_resource.trigger( + "PUT", + send_msg_path, + msg_content + ) + self.assertEquals(403, code, msg=str(response)) + + # send message in created room and joined, expect 200 + yield self.join(room=self.created_rmid, user=self.user_id) + (code, response) = yield self.mock_resource.trigger( + "PUT", + send_msg_path, + msg_content + ) + self.assertEquals(200, code, msg=str(response)) + + # send message in created room and left, expect 403 + yield self.leave(room=self.created_rmid, user=self.user_id) + (code, response) = yield self.mock_resource.trigger( + "PUT", + send_msg_path, + msg_content + ) + self.assertEquals(403, code, msg=str(response)) + + @defer.inlineCallbacks + def test_topic_perms(self): + topic_content = '{"topic":"My Topic Name"}' + topic_path = "/rooms/%s/state/m.room.topic" % self.created_rmid + + # set/get topic in uncreated room, expect 403 + (code, response) = yield self.mock_resource.trigger( + "PUT", "/rooms/%s/state/m.room.topic" % self.uncreated_rmid, + topic_content) + self.assertEquals(403, code, msg=str(response)) + (code, response) = yield self.mock_resource.trigger_get( + "/rooms/%s/state/m.room.topic" % self.uncreated_rmid) + self.assertEquals(403, code, msg=str(response)) + + # set/get topic in created PRIVATE room not joined, expect 403 + (code, response) = yield self.mock_resource.trigger( + "PUT", topic_path, topic_content) + self.assertEquals(403, code, msg=str(response)) + (code, response) = yield self.mock_resource.trigger_get(topic_path) + self.assertEquals(403, code, msg=str(response)) + + # set topic in created PRIVATE room and invited, expect 403 + yield self.invite(room=self.created_rmid, src=self.rmcreator_id, + targ=self.user_id) + (code, response) = yield self.mock_resource.trigger( + "PUT", topic_path, topic_content) + self.assertEquals(403, code, msg=str(response)) + + # get topic in created PRIVATE room and invited, expect 403 + (code, response) = yield self.mock_resource.trigger_get(topic_path) + self.assertEquals(403, code, msg=str(response)) + + # set/get topic in created PRIVATE room and joined, expect 200 + yield self.join(room=self.created_rmid, user=self.user_id) + + # Only room ops can set topic by default + self.auth_user_id = self.rmcreator_id + (code, response) = yield self.mock_resource.trigger( + "PUT", topic_path, topic_content) + self.assertEquals(200, code, msg=str(response)) + self.auth_user_id = self.user_id + + (code, response) = yield self.mock_resource.trigger_get(topic_path) + self.assertEquals(200, code, msg=str(response)) + self.assert_dict(json.loads(topic_content), response) + + # set/get topic in created PRIVATE room and left, expect 403 + yield self.leave(room=self.created_rmid, user=self.user_id) + (code, response) = yield self.mock_resource.trigger( + "PUT", topic_path, topic_content) + self.assertEquals(403, code, msg=str(response)) + (code, response) = yield self.mock_resource.trigger_get(topic_path) + self.assertEquals(403, code, msg=str(response)) + + # get topic in PUBLIC room, not joined, expect 403 + (code, response) = yield self.mock_resource.trigger_get( + "/rooms/%s/state/m.room.topic" % self.created_public_rmid) + self.assertEquals(403, code, msg=str(response)) + + # set topic in PUBLIC room, not joined, expect 403 + (code, response) = yield self.mock_resource.trigger( + "PUT", + "/rooms/%s/state/m.room.topic" % self.created_public_rmid, + topic_content) + self.assertEquals(403, code, msg=str(response)) + + @defer.inlineCallbacks + def _test_get_membership(self, room=None, members=[], expect_code=None): + path = "/rooms/%s/state/m.room.member/%s" + for member in members: + (code, response) = yield self.mock_resource.trigger_get( + path % + (room, member)) + self.assertEquals(expect_code, code) + + @defer.inlineCallbacks + def test_membership_basic_room_perms(self): + # === room does not exist === + room = self.uncreated_rmid + # get membership of self, get membership of other, uncreated room + # expect all 403s + yield self._test_get_membership( + members=[self.user_id, self.rmcreator_id], + room=room, expect_code=403) + + # trying to invite people to this room should 403 + yield self.invite(room=room, src=self.user_id, targ=self.rmcreator_id, + expect_code=403) + + # set [invite/join/left] of self, set [invite/join/left] of other, + # expect all 403s + for usr in [self.user_id, self.rmcreator_id]: + yield self.join(room=room, user=usr, expect_code=404) + yield self.leave(room=room, user=usr, expect_code=403) + + @defer.inlineCallbacks + def test_membership_private_room_perms(self): + room = self.created_rmid + # get membership of self, get membership of other, private room + invite + # expect all 403s + yield self.invite(room=room, src=self.rmcreator_id, + targ=self.user_id) + yield self._test_get_membership( + members=[self.user_id, self.rmcreator_id], + room=room, expect_code=403) + + # get membership of self, get membership of other, private room + joined + # expect all 200s + yield self.join(room=room, user=self.user_id) + yield self._test_get_membership( + members=[self.user_id, self.rmcreator_id], + room=room, expect_code=200) + + # get membership of self, get membership of other, private room + left + # expect all 403s + yield self.leave(room=room, user=self.user_id) + yield self._test_get_membership( + members=[self.user_id, self.rmcreator_id], + room=room, expect_code=403) + + @defer.inlineCallbacks + def test_membership_public_room_perms(self): + room = self.created_public_rmid + # get membership of self, get membership of other, public room + invite + # expect 403 + yield self.invite(room=room, src=self.rmcreator_id, + targ=self.user_id) + yield self._test_get_membership( + members=[self.user_id, self.rmcreator_id], + room=room, expect_code=403) + + # get membership of self, get membership of other, public room + joined + # expect all 200s + yield self.join(room=room, user=self.user_id) + yield self._test_get_membership( + members=[self.user_id, self.rmcreator_id], + room=room, expect_code=200) + + # get membership of self, get membership of other, public room + left + # expect 403. + yield self.leave(room=room, user=self.user_id) + yield self._test_get_membership( + members=[self.user_id, self.rmcreator_id], + room=room, expect_code=403) + + @defer.inlineCallbacks + def test_invited_permissions(self): + room = self.created_rmid + yield self.invite(room=room, src=self.rmcreator_id, targ=self.user_id) + + # set [invite/join/left] of other user, expect 403s + yield self.invite(room=room, src=self.user_id, targ=self.rmcreator_id, + expect_code=403) + yield self.change_membership(room=room, src=self.user_id, + targ=self.rmcreator_id, + membership=Membership.JOIN, + expect_code=403) + yield self.change_membership(room=room, src=self.user_id, + targ=self.rmcreator_id, + membership=Membership.LEAVE, + expect_code=403) + + @defer.inlineCallbacks + def test_joined_permissions(self): + room = self.created_rmid + yield self.invite(room=room, src=self.rmcreator_id, targ=self.user_id) + yield self.join(room=room, user=self.user_id) + + # set invited of self, expect 403 + yield self.invite(room=room, src=self.user_id, targ=self.user_id, + expect_code=403) + + # set joined of self, expect 200 (NOOP) + yield self.join(room=room, user=self.user_id) + + other = "@burgundy:red" + # set invited of other, expect 200 + yield self.invite(room=room, src=self.user_id, targ=other, + expect_code=200) + + # set joined of other, expect 403 + yield self.change_membership(room=room, src=self.user_id, + targ=other, + membership=Membership.JOIN, + expect_code=403) + + # set left of other, expect 403 + yield self.change_membership(room=room, src=self.user_id, + targ=other, + membership=Membership.LEAVE, + expect_code=403) + + # set left of self, expect 200 + yield self.leave(room=room, user=self.user_id) + + @defer.inlineCallbacks + def test_leave_permissions(self): + room = self.created_rmid + yield self.invite(room=room, src=self.rmcreator_id, targ=self.user_id) + yield self.join(room=room, user=self.user_id) + yield self.leave(room=room, user=self.user_id) + + # set [invite/join/left] of self, set [invite/join/left] of other, + # expect all 403s + for usr in [self.user_id, self.rmcreator_id]: + yield self.change_membership( + room=room, + src=self.user_id, + targ=usr, + membership=Membership.INVITE, + expect_code=403 + ) + + yield self.change_membership( + room=room, + src=self.user_id, + targ=usr, + membership=Membership.JOIN, + expect_code=403 + ) + + # It is always valid to LEAVE if you've already left (currently.) + yield self.change_membership( + room=room, + src=self.user_id, + targ=self.rmcreator_id, + membership=Membership.LEAVE, + expect_code=403 + ) + + +class RoomsMemberListTestCase(RestTestCase): + """ Tests /rooms/$room_id/members/list REST events.""" + user_id = "@sid1:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "red", + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + self.auth_user_id = self.user_id + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + + synapse.client.v1.room.register_servlets(hs, self.mock_resource) + + def tearDown(self): + pass + + @defer.inlineCallbacks + def test_get_member_list(self): + room_id = yield self.create_room_as(self.user_id) + (code, response) = yield self.mock_resource.trigger_get( + "/rooms/%s/members" % room_id) + self.assertEquals(200, code, msg=str(response)) + + @defer.inlineCallbacks + def test_get_member_list_no_room(self): + (code, response) = yield self.mock_resource.trigger_get( + "/rooms/roomdoesnotexist/members") + self.assertEquals(403, code, msg=str(response)) + + @defer.inlineCallbacks + def test_get_member_list_no_permission(self): + room_id = yield self.create_room_as("@some_other_guy:red") + (code, response) = yield self.mock_resource.trigger_get( + "/rooms/%s/members" % room_id) + self.assertEquals(403, code, msg=str(response)) + + @defer.inlineCallbacks + def test_get_member_list_mixed_memberships(self): + room_creator = "@some_other_guy:red" + room_id = yield self.create_room_as(room_creator) + room_path = "/rooms/%s/members" % room_id + yield self.invite(room=room_id, src=room_creator, + targ=self.user_id) + # can't see list if you're just invited. + (code, response) = yield self.mock_resource.trigger_get(room_path) + self.assertEquals(403, code, msg=str(response)) + + yield self.join(room=room_id, user=self.user_id) + # can see list now joined + (code, response) = yield self.mock_resource.trigger_get(room_path) + self.assertEquals(200, code, msg=str(response)) + + yield self.leave(room=room_id, user=self.user_id) + # can no longer see list, you've left. + (code, response) = yield self.mock_resource.trigger_get(room_path) + self.assertEquals(403, code, msg=str(response)) + + +class RoomsCreateTestCase(RestTestCase): + """ Tests /rooms and /rooms/$room_id REST events. """ + user_id = "@sid1:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.auth_user_id = self.user_id + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "red", + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + + synapse.client.v1.room.register_servlets(hs, self.mock_resource) + + def tearDown(self): + pass + + @defer.inlineCallbacks + def test_post_room_no_keys(self): + # POST with no config keys, expect new room id + (code, response) = yield self.mock_resource.trigger("POST", + "/createRoom", + "{}") + self.assertEquals(200, code, response) + self.assertTrue("room_id" in response) + + @defer.inlineCallbacks + def test_post_room_visibility_key(self): + # POST with visibility config key, expect new room id + (code, response) = yield self.mock_resource.trigger( + "POST", + "/createRoom", + '{"visibility":"private"}') + self.assertEquals(200, code) + self.assertTrue("room_id" in response) + + @defer.inlineCallbacks + def test_post_room_custom_key(self): + # POST with custom config keys, expect new room id + (code, response) = yield self.mock_resource.trigger( + "POST", + "/createRoom", + '{"custom":"stuff"}') + self.assertEquals(200, code) + self.assertTrue("room_id" in response) + + @defer.inlineCallbacks + def test_post_room_known_and_unknown_keys(self): + # POST with custom + known config keys, expect new room id + (code, response) = yield self.mock_resource.trigger( + "POST", + "/createRoom", + '{"visibility":"private","custom":"things"}') + self.assertEquals(200, code) + self.assertTrue("room_id" in response) + + @defer.inlineCallbacks + def test_post_room_invalid_content(self): + # POST with invalid content / paths, expect 400 + (code, response) = yield self.mock_resource.trigger( + "POST", + "/createRoom", + '{"visibili') + self.assertEquals(400, code) + + (code, response) = yield self.mock_resource.trigger( + "POST", + "/createRoom", + '["hello"]') + self.assertEquals(400, code) + + +class RoomTopicTestCase(RestTestCase): + """ Tests /rooms/$room_id/topic REST events. """ + user_id = "@sid1:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.auth_user_id = self.user_id + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "red", + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + + hs.get_auth().get_user_by_token = _get_user_by_token + + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + + synapse.client.v1.room.register_servlets(hs, self.mock_resource) + + # create the room + self.room_id = yield self.create_room_as(self.user_id) + self.path = "/rooms/%s/state/m.room.topic" % (self.room_id,) + + def tearDown(self): + pass + + @defer.inlineCallbacks + def test_invalid_puts(self): + # missing keys or invalid json + (code, response) = yield self.mock_resource.trigger("PUT", + self.path, '{}') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + self.path, '{"_name":"bob"}') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + self.path, '{"nao') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + self.path, '[{"_name":"bob"},{"_name":"jill"}]') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + self.path, 'text only') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + self.path, '') + self.assertEquals(400, code, msg=str(response)) + + # valid key, wrong type + content = '{"topic":["Topic name"]}' + (code, response) = yield self.mock_resource.trigger("PUT", + self.path, content) + self.assertEquals(400, code, msg=str(response)) + + @defer.inlineCallbacks + def test_rooms_topic(self): + # nothing should be there + (code, response) = yield self.mock_resource.trigger_get(self.path) + self.assertEquals(404, code, msg=str(response)) + + # valid put + content = '{"topic":"Topic name"}' + (code, response) = yield self.mock_resource.trigger("PUT", + self.path, content) + self.assertEquals(200, code, msg=str(response)) + + # valid get + (code, response) = yield self.mock_resource.trigger_get(self.path) + self.assertEquals(200, code, msg=str(response)) + self.assert_dict(json.loads(content), response) + + @defer.inlineCallbacks + def test_rooms_topic_with_extra_keys(self): + # valid put with extra keys + content = '{"topic":"Seasons","subtopic":"Summer"}' + (code, response) = yield self.mock_resource.trigger("PUT", + self.path, content) + self.assertEquals(200, code, msg=str(response)) + + # valid get + (code, response) = yield self.mock_resource.trigger_get(self.path) + self.assertEquals(200, code, msg=str(response)) + self.assert_dict(json.loads(content), response) + + +class RoomMemberStateTestCase(RestTestCase): + """ Tests /rooms/$room_id/members/$user_id/state REST events. """ + user_id = "@sid1:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.auth_user_id = self.user_id + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "red", + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + + synapse.client.v1.room.register_servlets(hs, self.mock_resource) + + self.room_id = yield self.create_room_as(self.user_id) + + def tearDown(self): + pass + + @defer.inlineCallbacks + def test_invalid_puts(self): + path = "/rooms/%s/state/m.room.member/%s" % (self.room_id, self.user_id) + # missing keys or invalid json + (code, response) = yield self.mock_resource.trigger("PUT", + path, '{}') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + path, '{"_name":"bob"}') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + path, '{"nao') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + path, '[{"_name":"bob"},{"_name":"jill"}]') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + path, 'text only') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + path, '') + self.assertEquals(400, code, msg=str(response)) + + # valid keys, wrong types + content = ('{"membership":["%s","%s","%s"]}' % + (Membership.INVITE, Membership.JOIN, Membership.LEAVE)) + (code, response) = yield self.mock_resource.trigger("PUT", path, content) + self.assertEquals(400, code, msg=str(response)) + + @defer.inlineCallbacks + def test_rooms_members_self(self): + path = "/rooms/%s/state/m.room.member/%s" % ( + urllib.quote(self.room_id), self.user_id + ) + + # valid join message (NOOP since we made the room) + content = '{"membership":"%s"}' % Membership.JOIN + (code, response) = yield self.mock_resource.trigger("PUT", path, content) + self.assertEquals(200, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("GET", path, None) + self.assertEquals(200, code, msg=str(response)) + + expected_response = { + "membership": Membership.JOIN, + } + self.assertEquals(expected_response, response) + + @defer.inlineCallbacks + def test_rooms_members_other(self): + self.other_id = "@zzsid1:red" + path = "/rooms/%s/state/m.room.member/%s" % ( + urllib.quote(self.room_id), self.other_id + ) + + # valid invite message + content = '{"membership":"%s"}' % Membership.INVITE + (code, response) = yield self.mock_resource.trigger("PUT", path, content) + self.assertEquals(200, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("GET", path, None) + self.assertEquals(200, code, msg=str(response)) + self.assertEquals(json.loads(content), response) + + @defer.inlineCallbacks + def test_rooms_members_other_custom_keys(self): + self.other_id = "@zzsid1:red" + path = "/rooms/%s/state/m.room.member/%s" % ( + urllib.quote(self.room_id), self.other_id + ) + + # valid invite message with custom key + content = ('{"membership":"%s","invite_text":"%s"}' % + (Membership.INVITE, "Join us!")) + (code, response) = yield self.mock_resource.trigger("PUT", path, content) + self.assertEquals(200, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("GET", path, None) + self.assertEquals(200, code, msg=str(response)) + self.assertEquals(json.loads(content), response) + + +class RoomMessagesTestCase(RestTestCase): + """ Tests /rooms/$room_id/messages/$user_id/$msg_id REST events. """ + user_id = "@sid1:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.auth_user_id = self.user_id + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "red", + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + + synapse.client.v1.room.register_servlets(hs, self.mock_resource) + + self.room_id = yield self.create_room_as(self.user_id) + + def tearDown(self): + pass + + @defer.inlineCallbacks + def test_invalid_puts(self): + path = "/rooms/%s/send/m.room.message/mid1" % ( + urllib.quote(self.room_id)) + # missing keys or invalid json + (code, response) = yield self.mock_resource.trigger("PUT", + path, '{}') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + path, '{"_name":"bob"}') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + path, '{"nao') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + path, '[{"_name":"bob"},{"_name":"jill"}]') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + path, 'text only') + self.assertEquals(400, code, msg=str(response)) + + (code, response) = yield self.mock_resource.trigger("PUT", + path, '') + self.assertEquals(400, code, msg=str(response)) + + @defer.inlineCallbacks + def test_rooms_messages_sent(self): + path = "/rooms/%s/send/m.room.message/mid1" % ( + urllib.quote(self.room_id)) + + content = '{"body":"test","msgtype":{"type":"a"}}' + (code, response) = yield self.mock_resource.trigger("PUT", path, content) + self.assertEquals(400, code, msg=str(response)) + + # custom message types + content = '{"body":"test","msgtype":"test.custom.text"}' + (code, response) = yield self.mock_resource.trigger("PUT", path, content) + self.assertEquals(200, code, msg=str(response)) + +# (code, response) = yield self.mock_resource.trigger("GET", path, None) +# self.assertEquals(200, code, msg=str(response)) +# self.assert_dict(json.loads(content), response) + + # m.text message type + path = "/rooms/%s/send/m.room.message/mid2" % ( + urllib.quote(self.room_id)) + content = '{"body":"test2","msgtype":"m.text"}' + (code, response) = yield self.mock_resource.trigger("PUT", path, content) + self.assertEquals(200, code, msg=str(response)) + + +class RoomInitialSyncTestCase(RestTestCase): + """ Tests /rooms/$room_id/initialSync. """ + user_id = "@sid1:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.auth_user_id = self.user_id + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "red", + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + + synapse.client.v1.room.register_servlets(hs, self.mock_resource) + + # Since I'm getting my own presence I need to exist as far as presence + # is concerned. + hs.get_handlers().presence_handler.registered_user( + hs.parse_userid(self.user_id) + ) + + # create the room + self.room_id = yield self.create_room_as(self.user_id) + + @defer.inlineCallbacks + def test_initial_sync(self): + (code, response) = yield self.mock_resource.trigger_get( + "/rooms/%s/initialSync" % self.room_id) + self.assertEquals(200, code) + + self.assertEquals(self.room_id, response["room_id"]) + self.assertEquals("join", response["membership"]) + + # Room state is easier to assert on if we unpack it into a dict + state = {} + for event in response["state"]: + if "state_key" not in event: + continue + t = event["type"] + if t not in state: + state[t] = [] + state[t].append(event) + + self.assertTrue("m.room.create" in state) + + self.assertTrue("messages" in response) + self.assertTrue("chunk" in response["messages"]) + self.assertTrue("end" in response["messages"]) + + self.assertTrue("presence" in response) + + presence_by_user = {e["content"]["user_id"]: e + for e in response["presence"] + } + self.assertTrue(self.user_id in presence_by_user) + self.assertEquals("m.presence", presence_by_user[self.user_id]["type"]) diff --git a/tests/client/v1/test_typing.py b/tests/client/v1/test_typing.py new file mode 100644 index 0000000000..d6d677bde3 --- /dev/null +++ b/tests/client/v1/test_typing.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests REST events for /rooms paths.""" + +# twisted imports +from twisted.internet import defer + +import synapse.client.v1.room +from synapse.server import HomeServer + +from ...utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey +from .utils import RestTestCase + +from mock import Mock, NonCallableMock + + +PATH_PREFIX = "/_matrix/client/api/v1" + + +class RoomTypingTestCase(RestTestCase): + """ Tests /rooms/$room_id/typing/$user_id REST API. """ + user_id = "@sid:red" + + @defer.inlineCallbacks + def setUp(self): + self.clock = MockClock() + + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.auth_user_id = self.user_id + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "red", + clock=self.clock, + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.hs = hs + + self.event_source = hs.get_event_sources().sources["typing"] + + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + + hs.get_auth().get_user_by_token = _get_user_by_token + + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + + def get_room_members(room_id): + if room_id == self.room_id: + return defer.succeed([hs.parse_userid(self.user_id)]) + else: + return defer.succeed([]) + + @defer.inlineCallbacks + def fetch_room_distributions_into(room_id, localusers=None, + remotedomains=None, ignore_user=None): + + members = yield get_room_members(room_id) + for member in members: + if ignore_user is not None and member == ignore_user: + continue + + if hs.is_mine(member): + if localusers is not None: + localusers.add(member) + else: + if remotedomains is not None: + remotedomains.add(member.domain) + hs.get_handlers().room_member_handler.fetch_room_distributions_into = ( + fetch_room_distributions_into) + + synapse.client.v1.room.register_servlets(hs, self.mock_resource) + + self.room_id = yield self.create_room_as(self.user_id) + # Need another user to make notifications actually work + yield self.join(self.room_id, user="@jim:red") + + def tearDown(self): + self.hs.get_handlers().typing_notification_handler.tearDown() + + @defer.inlineCallbacks + def test_set_typing(self): + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": true, "timeout": 30000}' + ) + self.assertEquals(200, code) + + self.assertEquals(self.event_source.get_current_key(), 1) + self.assertEquals( + self.event_source.get_new_events_for_user(self.user_id, 0, None)[0], + [ + {"type": "m.typing", + "room_id": self.room_id, + "content": { + "user_ids": [self.user_id], + }}, + ] + ) + + @defer.inlineCallbacks + def test_set_not_typing(self): + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": false}' + ) + self.assertEquals(200, code) + + @defer.inlineCallbacks + def test_typing_timeout(self): + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": true, "timeout": 30000}' + ) + self.assertEquals(200, code) + + self.assertEquals(self.event_source.get_current_key(), 1) + + self.clock.advance_time(31); + + self.assertEquals(self.event_source.get_current_key(), 2) + + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": true, "timeout": 30000}' + ) + self.assertEquals(200, code) + + self.assertEquals(self.event_source.get_current_key(), 3) diff --git a/tests/client/v1/utils.py b/tests/client/v1/utils.py new file mode 100644 index 0000000000..579441fb4a --- /dev/null +++ b/tests/client/v1/utils.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# twisted imports +from twisted.internet import defer + +# trial imports +from tests import unittest + +from synapse.api.constants import Membership + +import json +import time + + +class RestTestCase(unittest.TestCase): + """Contains extra helper functions to quickly and clearly perform a given + REST action, which isn't the focus of the test. + + This subclass assumes there are mock_resource and auth_user_id attributes. + """ + + def __init__(self, *args, **kwargs): + super(RestTestCase, self).__init__(*args, **kwargs) + self.mock_resource = None + self.auth_user_id = None + + def mock_get_user_by_token(self, token=None): + return self.auth_user_id + + @defer.inlineCallbacks + def create_room_as(self, room_creator, is_public=True, tok=None): + temp_id = self.auth_user_id + self.auth_user_id = room_creator + path = "/createRoom" + content = "{}" + if not is_public: + content = '{"visibility":"private"}' + if tok: + path = path + "?access_token=%s" % tok + (code, response) = yield self.mock_resource.trigger("POST", path, content) + self.assertEquals(200, code, msg=str(response)) + self.auth_user_id = temp_id + defer.returnValue(response["room_id"]) + + @defer.inlineCallbacks + def invite(self, room=None, src=None, targ=None, expect_code=200, tok=None): + yield self.change_membership(room=room, src=src, targ=targ, tok=tok, + membership=Membership.INVITE, + expect_code=expect_code) + + @defer.inlineCallbacks + def join(self, room=None, user=None, expect_code=200, tok=None): + yield self.change_membership(room=room, src=user, targ=user, tok=tok, + membership=Membership.JOIN, + expect_code=expect_code) + + @defer.inlineCallbacks + def leave(self, room=None, user=None, expect_code=200, tok=None): + yield self.change_membership(room=room, src=user, targ=user, tok=tok, + membership=Membership.LEAVE, + expect_code=expect_code) + + @defer.inlineCallbacks + def change_membership(self, room, src, targ, membership, tok=None, + expect_code=200): + temp_id = self.auth_user_id + self.auth_user_id = src + + path = "/rooms/%s/state/m.room.member/%s" % (room, targ) + if tok: + path = path + "?access_token=%s" % tok + + data = { + "membership": membership + } + + (code, response) = yield self.mock_resource.trigger("PUT", path, + json.dumps(data)) + self.assertEquals(expect_code, code, msg=str(response)) + + self.auth_user_id = temp_id + + @defer.inlineCallbacks + def register(self, user_id): + (code, response) = yield self.mock_resource.trigger( + "POST", + "/register", + json.dumps({ + "user": user_id, + "password": "test", + "type": "m.login.password" + })) + self.assertEquals(200, code) + defer.returnValue(response) + + @defer.inlineCallbacks + def send(self, room_id, body=None, txn_id=None, tok=None, + expect_code=200): + if txn_id is None: + txn_id = "m%s" % (str(time.time())) + if body is None: + body = "body_text_here" + + path = "/rooms/%s/send/m.room.message/%s" % (room_id, txn_id) + content = '{"msgtype":"m.text","body":"%s"}' % body + if tok: + path = path + "?access_token=%s" % tok + + (code, response) = yield self.mock_resource.trigger("PUT", path, content) + self.assertEquals(expect_code, code, msg=str(response)) + + def assert_dict(self, required, actual): + """Does a partial assert of a dict. + + Args: + required (dict): The keys and value which MUST be in 'actual'. + actual (dict): The test result. Extra keys will not be checked. + """ + for key in required: + self.assertEquals(required[key], actual[key], + msg="%s mismatch. %s" % (key, actual)) diff --git a/tests/rest/__init__.py b/tests/rest/__init__.py deleted file mode 100644 index 9bff9ec169..0000000000 --- a/tests/rest/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py deleted file mode 100644 index d3159e2cf4..0000000000 --- a/tests/rest/test_events.py +++ /dev/null @@ -1,225 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Tests REST events for /events paths.""" -from tests import unittest - -# twisted imports -from twisted.internet import defer - -import synapse.rest.events -import synapse.rest.register -import synapse.rest.room - -from synapse.server import HomeServer - -from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey -from .utils import RestTestCase - -from mock import Mock, NonCallableMock - - -PATH_PREFIX = "/_matrix/client/api/v1" - - -class EventStreamPaginationApiTestCase(unittest.TestCase): - """ Tests event streaming query parameters and start/end keys used in the - Pagination stream API. """ - user_id = "sid1" - - def setUp(self): - # configure stream and inject items - pass - - def tearDown(self): - pass - - def TODO_test_long_poll(self): - # stream from 'end' key, send (self+other) message, expect message. - - # stream from 'END', send (self+other) message, expect message. - - # stream from 'end' key, send (self+other) topic, expect topic. - - # stream from 'END', send (self+other) topic, expect topic. - - # stream from 'end' key, send (self+other) invite, expect invite. - - # stream from 'END', send (self+other) invite, expect invite. - - pass - - def TODO_test_stream_forward(self): - # stream from START, expect injected items - - # stream from 'start' key, expect same content - - # stream from 'end' key, expect nothing - - # stream from 'END', expect nothing - - # The following is needed for cases where content is removed e.g. you - # left a room, so the token you're streaming from is > the one that - # would be returned naturally from START>END. - # stream from very new token (higher than end key), expect same token - # returned as end key - pass - - def TODO_test_limits(self): - # stream from a key, expect limit_num items - - # stream from START, expect limit_num items - - pass - - def TODO_test_range(self): - # stream from key to key, expect X items - - # stream from key to END, expect X items - - # stream from START to key, expect X items - - # stream from START to END, expect all items - pass - - def TODO_test_direction(self): - # stream from END to START and fwds, expect newest first - - # stream from END to START and bwds, expect oldest first - - # stream from START to END and fwds, expect oldest first - - # stream from START to END and bwds, expect newest first - - pass - - -class EventStreamPermissionsTestCase(RestTestCase): - """ Tests event streaming (GET /events). """ - - @defer.inlineCallbacks - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "test", - db_pool=db_pool, - http_client=None, - replication_layer=Mock(), - clock=Mock(spec=[ - "call_later", - "cancel_call_later", - "time_msec", - "time" - ]), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, - ) - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - hs.config.enable_registration_captcha = False - - hs.get_handlers().federation_handler = Mock() - - hs.get_clock().time_msec.return_value = 1000000 - hs.get_clock().time.return_value = 1000 - - synapse.rest.register.register_servlets(hs, self.mock_resource) - synapse.rest.events.register_servlets(hs, self.mock_resource) - synapse.rest.room.register_servlets(hs, self.mock_resource) - - # register an account - self.user_id = "sid1" - response = yield self.register(self.user_id) - self.token = response["access_token"] - self.user_id = response["user_id"] - - # register a 2nd account - self.other_user = "other1" - response = yield self.register(self.other_user) - self.other_token = response["access_token"] - self.other_user = response["user_id"] - - def tearDown(self): - pass - - @defer.inlineCallbacks - def test_stream_basic_permissions(self): - # invalid token, expect 403 - (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s" % ("invalid" + self.token, ) - ) - self.assertEquals(403, code, msg=str(response)) - - # valid token, expect content - (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s&timeout=0" % (self.token,) - ) - self.assertEquals(200, code, msg=str(response)) - self.assertTrue("chunk" in response) - self.assertTrue("start" in response) - self.assertTrue("end" in response) - - @defer.inlineCallbacks - def test_stream_room_permissions(self): - room_id = yield self.create_room_as( - self.other_user, - tok=self.other_token - ) - yield self.send(room_id, tok=self.other_token) - - # invited to room (expect no content for room) - yield self.invite( - room_id, - src=self.other_user, - targ=self.user_id, - tok=self.other_token - ) - - (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s&timeout=0" % (self.token,) - ) - self.assertEquals(200, code, msg=str(response)) - - self.assertEquals(0, len(response["chunk"])) - - # joined room (expect all content for room) - yield self.join(room=room_id, user=self.user_id, tok=self.token) - - # left to room (expect no content for room) - - def TODO_test_stream_items(self): - # new user, no content - - # join room, expect 1 item (join) - - # send message, expect 2 items (join,send) - - # set topic, expect 3 items (join,send,topic) - - # someone else join room, expect 4 (join,send,topic,join) - - # someone else send message, expect 5 (join,send.topic,join,send) - - # someone else set topic, expect 6 (join,send,topic,join,send,topic) - pass diff --git a/tests/rest/test_presence.py b/tests/rest/test_presence.py deleted file mode 100644 index 769c7824bc..0000000000 --- a/tests/rest/test_presence.py +++ /dev/null @@ -1,372 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests REST events for /presence paths.""" - -from tests import unittest -from twisted.internet import defer - -from mock import Mock - -from ..utils import MockHttpResource, MockKey - -from synapse.api.constants import PresenceState -from synapse.handlers.presence import PresenceHandler -from synapse.server import HomeServer - - -OFFLINE = PresenceState.OFFLINE -UNAVAILABLE = PresenceState.UNAVAILABLE -ONLINE = PresenceState.ONLINE - - -myid = "@apple:test" -PATH_PREFIX = "/_matrix/client/api/v1" - - -class JustPresenceHandlers(object): - def __init__(self, hs): - self.presence_handler = PresenceHandler(hs) - - -class PresenceStateTestCase(unittest.TestCase): - - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - hs = HomeServer("test", - db_pool=None, - datastore=Mock(spec=[ - "get_presence_state", - "set_presence_state", - "insert_client_ip", - ]), - http_client=None, - resource_for_client=self.mock_resource, - resource_for_federation=self.mock_resource, - config=self.mock_config, - ) - hs.handlers = JustPresenceHandlers(hs) - - self.datastore = hs.get_datastore() - - def get_presence_list(*a, **kw): - return defer.succeed([]) - self.datastore.get_presence_list = get_presence_list - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(myid), - "admin": False, - "device_id": None, - } - - hs.get_auth().get_user_by_token = _get_user_by_token - - room_member_handler = hs.handlers.room_member_handler = Mock( - spec=[ - "get_rooms_for_user", - ] - ) - - def get_rooms_for_user(user): - return defer.succeed([]) - room_member_handler.get_rooms_for_user = get_rooms_for_user - - hs.register_servlets() - - self.u_apple = hs.parse_userid(myid) - - @defer.inlineCallbacks - def test_get_my_status(self): - mocked_get = self.datastore.get_presence_state - mocked_get.return_value = defer.succeed( - {"state": ONLINE, "status_msg": "Available"} - ) - - (code, response) = yield self.mock_resource.trigger("GET", - "/presence/%s/status" % (myid), None) - - self.assertEquals(200, code) - self.assertEquals( - {"presence": ONLINE, "status_msg": "Available"}, - response - ) - mocked_get.assert_called_with("apple") - - @defer.inlineCallbacks - def test_set_my_status(self): - mocked_set = self.datastore.set_presence_state - mocked_set.return_value = defer.succeed({"state": OFFLINE}) - - (code, response) = yield self.mock_resource.trigger("PUT", - "/presence/%s/status" % (myid), - '{"presence": "unavailable", "status_msg": "Away"}') - - self.assertEquals(200, code) - mocked_set.assert_called_with("apple", - {"state": UNAVAILABLE, "status_msg": "Away"} - ) - - -class PresenceListTestCase(unittest.TestCase): - - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer("test", - db_pool=None, - datastore=Mock(spec=[ - "has_presence_state", - "get_presence_state", - "allow_presence_visible", - "is_presence_visible", - "add_presence_list_pending", - "set_presence_list_accepted", - "del_presence_list", - "get_presence_list", - "insert_client_ip", - ]), - http_client=None, - resource_for_client=self.mock_resource, - resource_for_federation=self.mock_resource, - config=self.mock_config, - ) - hs.handlers = JustPresenceHandlers(hs) - - self.datastore = hs.get_datastore() - - def has_presence_state(user_localpart): - return defer.succeed( - user_localpart in ("apple", "banana",) - ) - self.datastore.has_presence_state = has_presence_state - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(myid), - "admin": False, - "device_id": None, - } - - room_member_handler = hs.handlers.room_member_handler = Mock( - spec=[ - "get_rooms_for_user", - ] - ) - - hs.get_auth().get_user_by_token = _get_user_by_token - - hs.register_servlets() - - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - - @defer.inlineCallbacks - def test_get_my_list(self): - self.datastore.get_presence_list.return_value = defer.succeed( - [{"observed_user_id": "@banana:test"}], - ) - - (code, response) = yield self.mock_resource.trigger("GET", - "/presence/list/%s" % (myid), None) - - self.assertEquals(200, code) - self.assertEquals([ - {"user_id": "@banana:test", "presence": OFFLINE}, - ], response) - - self.datastore.get_presence_list.assert_called_with( - "apple", accepted=True - ) - - @defer.inlineCallbacks - def test_invite(self): - self.datastore.add_presence_list_pending.return_value = ( - defer.succeed(()) - ) - self.datastore.is_presence_visible.return_value = defer.succeed( - True - ) - - (code, response) = yield self.mock_resource.trigger("POST", - "/presence/list/%s" % (myid), - """{"invite": ["@banana:test"]}""" - ) - - self.assertEquals(200, code) - - self.datastore.add_presence_list_pending.assert_called_with( - "apple", "@banana:test" - ) - self.datastore.set_presence_list_accepted.assert_called_with( - "apple", "@banana:test" - ) - - @defer.inlineCallbacks - def test_drop(self): - self.datastore.del_presence_list.return_value = ( - defer.succeed(()) - ) - - (code, response) = yield self.mock_resource.trigger("POST", - "/presence/list/%s" % (myid), - """{"drop": ["@banana:test"]}""" - ) - - self.assertEquals(200, code) - - self.datastore.del_presence_list.assert_called_with( - "apple", "@banana:test" - ) - - -class PresenceEventStreamTestCase(unittest.TestCase): - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - - # HIDEOUS HACKERY - # TODO(paul): This should be injected in via the HomeServer DI system - from synapse.streams.events import ( - PresenceEventSource, NullSource, EventSources - ) - - old_SOURCE_TYPES = EventSources.SOURCE_TYPES - def tearDown(): - EventSources.SOURCE_TYPES = old_SOURCE_TYPES - self.tearDown = tearDown - - EventSources.SOURCE_TYPES = { - k: NullSource for k in old_SOURCE_TYPES.keys() - } - EventSources.SOURCE_TYPES["presence"] = PresenceEventSource - - hs = HomeServer("test", - db_pool=None, - http_client=None, - resource_for_client=self.mock_resource, - resource_for_federation=self.mock_resource, - datastore=Mock(spec=[ - "set_presence_state", - "get_presence_list", - ]), - clock=Mock(spec=[ - "call_later", - "cancel_call_later", - "time_msec", - ]), - config=self.mock_config, - ) - - hs.get_clock().time_msec.return_value = 1000000 - - def _get_user_by_req(req=None): - return hs.parse_userid(myid) - - hs.get_auth().get_user_by_req = _get_user_by_req - - hs.register_servlets() - - hs.handlers.room_member_handler = Mock(spec=[]) - - self.room_members = [] - - def get_rooms_for_user(user): - if user in self.room_members: - return ["a-room"] - else: - return [] - hs.handlers.room_member_handler.get_rooms_for_user = get_rooms_for_user - - self.mock_datastore = hs.get_datastore() - - def get_profile_displayname(user_id): - return defer.succeed("Frank") - self.mock_datastore.get_profile_displayname = get_profile_displayname - - def get_profile_avatar_url(user_id): - return defer.succeed(None) - self.mock_datastore.get_profile_avatar_url = get_profile_avatar_url - - def user_rooms_intersect(user_list): - room_member_ids = map(lambda u: u.to_string(), self.room_members) - - shared = all(map(lambda i: i in room_member_ids, user_list)) - return defer.succeed(shared) - self.mock_datastore.user_rooms_intersect = user_rooms_intersect - - def get_joined_hosts_for_room(room_id): - return [] - self.mock_datastore.get_joined_hosts_for_room = get_joined_hosts_for_room - - self.presence = hs.get_handlers().presence_handler - - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - - @defer.inlineCallbacks - def test_shortpoll(self): - self.room_members = [self.u_apple, self.u_banana] - - self.mock_datastore.set_presence_state.return_value = defer.succeed( - {"state": ONLINE} - ) - self.mock_datastore.get_presence_list.return_value = defer.succeed( - [] - ) - - (code, response) = yield self.mock_resource.trigger("GET", - "/events?timeout=0", None) - - self.assertEquals(200, code) - - # We've forced there to be only one data stream so the tokens will - # all be ours - - # I'll already get my own presence state change - self.assertEquals({"start": "0_1_0", "end": "0_1_0", "chunk": []}, - response - ) - - self.mock_datastore.set_presence_state.return_value = defer.succeed( - {"state": ONLINE} - ) - self.mock_datastore.get_presence_list.return_value = defer.succeed( - [] - ) - - yield self.presence.set_state(self.u_banana, self.u_banana, - state={"presence": ONLINE} - ) - - (code, response) = yield self.mock_resource.trigger("GET", - "/events?from=0_1_0&timeout=0", None) - - self.assertEquals(200, code) - self.assertEquals({"start": "0_1_0", "end": "0_2_0", "chunk": [ - {"type": "m.presence", - "content": { - "user_id": "@banana:test", - "presence": ONLINE, - "displayname": "Frank", - "last_active_ago": 0, - }}, - ]}, response) diff --git a/tests/rest/test_profile.py b/tests/rest/test_profile.py deleted file mode 100644 index 3a0d1e700a..0000000000 --- a/tests/rest/test_profile.py +++ /dev/null @@ -1,150 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests REST events for /profile paths.""" - -from tests import unittest -from twisted.internet import defer - -from mock import Mock, NonCallableMock - -from ..utils import MockHttpResource, MockKey - -from synapse.api.errors import SynapseError, AuthError -from synapse.server import HomeServer - -myid = "@1234ABCD:test" -PATH_PREFIX = "/_matrix/client/api/v1" - - -class ProfileTestCase(unittest.TestCase): - """ Tests profile management. """ - - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.mock_handler = Mock(spec=[ - "get_displayname", - "set_displayname", - "get_avatar_url", - "set_avatar_url", - ]) - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer("test", - db_pool=None, - http_client=None, - resource_for_client=self.mock_resource, - federation=Mock(), - replication_layer=Mock(), - datastore=None, - config=self.mock_config, - ) - - def _get_user_by_req(request=None): - return hs.parse_userid(myid) - - hs.get_auth().get_user_by_req = _get_user_by_req - - hs.get_handlers().profile_handler = self.mock_handler - - hs.register_servlets() - - @defer.inlineCallbacks - def test_get_my_name(self): - mocked_get = self.mock_handler.get_displayname - mocked_get.return_value = defer.succeed("Frank") - - (code, response) = yield self.mock_resource.trigger("GET", - "/profile/%s/displayname" % (myid), None) - - self.assertEquals(200, code) - self.assertEquals({"displayname": "Frank"}, response) - self.assertEquals(mocked_get.call_args[0][0].localpart, "1234ABCD") - - @defer.inlineCallbacks - def test_set_my_name(self): - mocked_set = self.mock_handler.set_displayname - mocked_set.return_value = defer.succeed(()) - - (code, response) = yield self.mock_resource.trigger("PUT", - "/profile/%s/displayname" % (myid), - '{"displayname": "Frank Jr."}') - - self.assertEquals(200, code) - self.assertEquals(mocked_set.call_args[0][0].localpart, "1234ABCD") - self.assertEquals(mocked_set.call_args[0][1].localpart, "1234ABCD") - self.assertEquals(mocked_set.call_args[0][2], "Frank Jr.") - - @defer.inlineCallbacks - def test_set_my_name_noauth(self): - mocked_set = self.mock_handler.set_displayname - mocked_set.side_effect = AuthError(400, "message") - - (code, response) = yield self.mock_resource.trigger("PUT", - "/profile/%s/displayname" % ("@4567:test"), '"Frank Jr."') - - self.assertTrue(400 <= code < 499, - msg="code %d is in the 4xx range" % (code)) - - @defer.inlineCallbacks - def test_get_other_name(self): - mocked_get = self.mock_handler.get_displayname - mocked_get.return_value = defer.succeed("Bob") - - (code, response) = yield self.mock_resource.trigger("GET", - "/profile/%s/displayname" % ("@opaque:elsewhere"), None) - - self.assertEquals(200, code) - self.assertEquals({"displayname": "Bob"}, response) - - @defer.inlineCallbacks - def test_set_other_name(self): - mocked_set = self.mock_handler.set_displayname - mocked_set.side_effect = SynapseError(400, "message") - - (code, response) = yield self.mock_resource.trigger("PUT", - "/profile/%s/displayname" % ("@opaque:elsewhere"), None) - - self.assertTrue(400 <= code <= 499, - msg="code %d is in the 4xx range" % (code)) - - @defer.inlineCallbacks - def test_get_my_avatar(self): - mocked_get = self.mock_handler.get_avatar_url - mocked_get.return_value = defer.succeed("http://my.server/me.png") - - (code, response) = yield self.mock_resource.trigger("GET", - "/profile/%s/avatar_url" % (myid), None) - - self.assertEquals(200, code) - self.assertEquals({"avatar_url": "http://my.server/me.png"}, response) - self.assertEquals(mocked_get.call_args[0][0].localpart, "1234ABCD") - - @defer.inlineCallbacks - def test_set_my_avatar(self): - mocked_set = self.mock_handler.set_avatar_url - mocked_set.return_value = defer.succeed(()) - - (code, response) = yield self.mock_resource.trigger("PUT", - "/profile/%s/avatar_url" % (myid), - '{"avatar_url": "http://my.server/pic.gif"}') - - self.assertEquals(200, code) - self.assertEquals(mocked_set.call_args[0][0].localpart, "1234ABCD") - self.assertEquals(mocked_set.call_args[0][1].localpart, "1234ABCD") - self.assertEquals(mocked_set.call_args[0][2], - "http://my.server/pic.gif") diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py deleted file mode 100644 index 8e65ff9a1c..0000000000 --- a/tests/rest/test_rooms.py +++ /dev/null @@ -1,1068 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests REST events for /rooms paths.""" - -# twisted imports -from twisted.internet import defer - -import synapse.rest.room -from synapse.api.constants import Membership - -from synapse.server import HomeServer - -from tests import unittest - -# python imports -import json -import urllib -import types - -from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey -from .utils import RestTestCase - -from mock import Mock, NonCallableMock - -PATH_PREFIX = "/_matrix/client/api/v1" - - -class RoomPermissionsTestCase(RestTestCase): - """ Tests room permissions. """ - user_id = "@sid1:red" - rmcreator_id = "@notme:red" - - @defer.inlineCallbacks - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "red", - db_pool=db_pool, - http_client=None, - replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, - ) - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - hs.get_handlers().federation_handler = Mock() - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(self.auth_user_id), - "admin": False, - "device_id": None, - } - hs.get_auth().get_user_by_token = _get_user_by_token - - def _insert_client_ip(*args, **kwargs): - return defer.succeed(None) - hs.get_datastore().insert_client_ip = _insert_client_ip - - self.auth_user_id = self.rmcreator_id - - synapse.rest.room.register_servlets(hs, self.mock_resource) - - self.auth = hs.get_auth() - - # create some rooms under the name rmcreator_id - self.uncreated_rmid = "!aa:test" - - self.created_rmid = yield self.create_room_as(self.rmcreator_id, - is_public=False) - - self.created_public_rmid = yield self.create_room_as(self.rmcreator_id, - is_public=True) - - # send a message in one of the rooms - self.created_rmid_msg_path = ("/rooms/%s/send/m.room.message/a1" % - (self.created_rmid)) - (code, response) = yield self.mock_resource.trigger( - "PUT", - self.created_rmid_msg_path, - '{"msgtype":"m.text","body":"test msg"}') - self.assertEquals(200, code, msg=str(response)) - - # set topic for public room - (code, response) = yield self.mock_resource.trigger( - "PUT", - "/rooms/%s/state/m.room.topic" % self.created_public_rmid, - '{"topic":"Public Room Topic"}') - self.assertEquals(200, code, msg=str(response)) - - # auth as user_id now - self.auth_user_id = self.user_id - - def tearDown(self): - pass - -# @defer.inlineCallbacks -# def test_get_message(self): -# # get message in uncreated room, expect 403 -# (code, response) = yield self.mock_resource.trigger_get( -# "/rooms/noroom/messages/someid/m1") -# self.assertEquals(403, code, msg=str(response)) -# -# # get message in created room not joined (no state), expect 403 -# (code, response) = yield self.mock_resource.trigger_get( -# self.created_rmid_msg_path) -# self.assertEquals(403, code, msg=str(response)) -# -# # get message in created room and invited, expect 403 -# yield self.invite(room=self.created_rmid, src=self.rmcreator_id, -# targ=self.user_id) -# (code, response) = yield self.mock_resource.trigger_get( -# self.created_rmid_msg_path) -# self.assertEquals(403, code, msg=str(response)) -# -# # get message in created room and joined, expect 200 -# yield self.join(room=self.created_rmid, user=self.user_id) -# (code, response) = yield self.mock_resource.trigger_get( -# self.created_rmid_msg_path) -# self.assertEquals(200, code, msg=str(response)) -# -# # get message in created room and left, expect 403 -# yield self.leave(room=self.created_rmid, user=self.user_id) -# (code, response) = yield self.mock_resource.trigger_get( -# self.created_rmid_msg_path) -# self.assertEquals(403, code, msg=str(response)) - - @defer.inlineCallbacks - def test_send_message(self): - msg_content = '{"msgtype":"m.text","body":"hello"}' - send_msg_path = ( - "/rooms/%s/send/m.room.message/mid1" % (self.created_rmid,) - ) - - # send message in uncreated room, expect 403 - (code, response) = yield self.mock_resource.trigger( - "PUT", - "/rooms/%s/send/m.room.message/mid2" % (self.uncreated_rmid,), - msg_content - ) - self.assertEquals(403, code, msg=str(response)) - - # send message in created room not joined (no state), expect 403 - (code, response) = yield self.mock_resource.trigger( - "PUT", - send_msg_path, - msg_content - ) - self.assertEquals(403, code, msg=str(response)) - - # send message in created room and invited, expect 403 - yield self.invite( - room=self.created_rmid, - src=self.rmcreator_id, - targ=self.user_id - ) - (code, response) = yield self.mock_resource.trigger( - "PUT", - send_msg_path, - msg_content - ) - self.assertEquals(403, code, msg=str(response)) - - # send message in created room and joined, expect 200 - yield self.join(room=self.created_rmid, user=self.user_id) - (code, response) = yield self.mock_resource.trigger( - "PUT", - send_msg_path, - msg_content - ) - self.assertEquals(200, code, msg=str(response)) - - # send message in created room and left, expect 403 - yield self.leave(room=self.created_rmid, user=self.user_id) - (code, response) = yield self.mock_resource.trigger( - "PUT", - send_msg_path, - msg_content - ) - self.assertEquals(403, code, msg=str(response)) - - @defer.inlineCallbacks - def test_topic_perms(self): - topic_content = '{"topic":"My Topic Name"}' - topic_path = "/rooms/%s/state/m.room.topic" % self.created_rmid - - # set/get topic in uncreated room, expect 403 - (code, response) = yield self.mock_resource.trigger( - "PUT", "/rooms/%s/state/m.room.topic" % self.uncreated_rmid, - topic_content) - self.assertEquals(403, code, msg=str(response)) - (code, response) = yield self.mock_resource.trigger_get( - "/rooms/%s/state/m.room.topic" % self.uncreated_rmid) - self.assertEquals(403, code, msg=str(response)) - - # set/get topic in created PRIVATE room not joined, expect 403 - (code, response) = yield self.mock_resource.trigger( - "PUT", topic_path, topic_content) - self.assertEquals(403, code, msg=str(response)) - (code, response) = yield self.mock_resource.trigger_get(topic_path) - self.assertEquals(403, code, msg=str(response)) - - # set topic in created PRIVATE room and invited, expect 403 - yield self.invite(room=self.created_rmid, src=self.rmcreator_id, - targ=self.user_id) - (code, response) = yield self.mock_resource.trigger( - "PUT", topic_path, topic_content) - self.assertEquals(403, code, msg=str(response)) - - # get topic in created PRIVATE room and invited, expect 403 - (code, response) = yield self.mock_resource.trigger_get(topic_path) - self.assertEquals(403, code, msg=str(response)) - - # set/get topic in created PRIVATE room and joined, expect 200 - yield self.join(room=self.created_rmid, user=self.user_id) - - # Only room ops can set topic by default - self.auth_user_id = self.rmcreator_id - (code, response) = yield self.mock_resource.trigger( - "PUT", topic_path, topic_content) - self.assertEquals(200, code, msg=str(response)) - self.auth_user_id = self.user_id - - (code, response) = yield self.mock_resource.trigger_get(topic_path) - self.assertEquals(200, code, msg=str(response)) - self.assert_dict(json.loads(topic_content), response) - - # set/get topic in created PRIVATE room and left, expect 403 - yield self.leave(room=self.created_rmid, user=self.user_id) - (code, response) = yield self.mock_resource.trigger( - "PUT", topic_path, topic_content) - self.assertEquals(403, code, msg=str(response)) - (code, response) = yield self.mock_resource.trigger_get(topic_path) - self.assertEquals(403, code, msg=str(response)) - - # get topic in PUBLIC room, not joined, expect 403 - (code, response) = yield self.mock_resource.trigger_get( - "/rooms/%s/state/m.room.topic" % self.created_public_rmid) - self.assertEquals(403, code, msg=str(response)) - - # set topic in PUBLIC room, not joined, expect 403 - (code, response) = yield self.mock_resource.trigger( - "PUT", - "/rooms/%s/state/m.room.topic" % self.created_public_rmid, - topic_content) - self.assertEquals(403, code, msg=str(response)) - - @defer.inlineCallbacks - def _test_get_membership(self, room=None, members=[], expect_code=None): - path = "/rooms/%s/state/m.room.member/%s" - for member in members: - (code, response) = yield self.mock_resource.trigger_get( - path % - (room, member)) - self.assertEquals(expect_code, code) - - @defer.inlineCallbacks - def test_membership_basic_room_perms(self): - # === room does not exist === - room = self.uncreated_rmid - # get membership of self, get membership of other, uncreated room - # expect all 403s - yield self._test_get_membership( - members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) - - # trying to invite people to this room should 403 - yield self.invite(room=room, src=self.user_id, targ=self.rmcreator_id, - expect_code=403) - - # set [invite/join/left] of self, set [invite/join/left] of other, - # expect all 403s - for usr in [self.user_id, self.rmcreator_id]: - yield self.join(room=room, user=usr, expect_code=404) - yield self.leave(room=room, user=usr, expect_code=403) - - @defer.inlineCallbacks - def test_membership_private_room_perms(self): - room = self.created_rmid - # get membership of self, get membership of other, private room + invite - # expect all 403s - yield self.invite(room=room, src=self.rmcreator_id, - targ=self.user_id) - yield self._test_get_membership( - members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) - - # get membership of self, get membership of other, private room + joined - # expect all 200s - yield self.join(room=room, user=self.user_id) - yield self._test_get_membership( - members=[self.user_id, self.rmcreator_id], - room=room, expect_code=200) - - # get membership of self, get membership of other, private room + left - # expect all 403s - yield self.leave(room=room, user=self.user_id) - yield self._test_get_membership( - members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) - - @defer.inlineCallbacks - def test_membership_public_room_perms(self): - room = self.created_public_rmid - # get membership of self, get membership of other, public room + invite - # expect 403 - yield self.invite(room=room, src=self.rmcreator_id, - targ=self.user_id) - yield self._test_get_membership( - members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) - - # get membership of self, get membership of other, public room + joined - # expect all 200s - yield self.join(room=room, user=self.user_id) - yield self._test_get_membership( - members=[self.user_id, self.rmcreator_id], - room=room, expect_code=200) - - # get membership of self, get membership of other, public room + left - # expect 403. - yield self.leave(room=room, user=self.user_id) - yield self._test_get_membership( - members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) - - @defer.inlineCallbacks - def test_invited_permissions(self): - room = self.created_rmid - yield self.invite(room=room, src=self.rmcreator_id, targ=self.user_id) - - # set [invite/join/left] of other user, expect 403s - yield self.invite(room=room, src=self.user_id, targ=self.rmcreator_id, - expect_code=403) - yield self.change_membership(room=room, src=self.user_id, - targ=self.rmcreator_id, - membership=Membership.JOIN, - expect_code=403) - yield self.change_membership(room=room, src=self.user_id, - targ=self.rmcreator_id, - membership=Membership.LEAVE, - expect_code=403) - - @defer.inlineCallbacks - def test_joined_permissions(self): - room = self.created_rmid - yield self.invite(room=room, src=self.rmcreator_id, targ=self.user_id) - yield self.join(room=room, user=self.user_id) - - # set invited of self, expect 403 - yield self.invite(room=room, src=self.user_id, targ=self.user_id, - expect_code=403) - - # set joined of self, expect 200 (NOOP) - yield self.join(room=room, user=self.user_id) - - other = "@burgundy:red" - # set invited of other, expect 200 - yield self.invite(room=room, src=self.user_id, targ=other, - expect_code=200) - - # set joined of other, expect 403 - yield self.change_membership(room=room, src=self.user_id, - targ=other, - membership=Membership.JOIN, - expect_code=403) - - # set left of other, expect 403 - yield self.change_membership(room=room, src=self.user_id, - targ=other, - membership=Membership.LEAVE, - expect_code=403) - - # set left of self, expect 200 - yield self.leave(room=room, user=self.user_id) - - @defer.inlineCallbacks - def test_leave_permissions(self): - room = self.created_rmid - yield self.invite(room=room, src=self.rmcreator_id, targ=self.user_id) - yield self.join(room=room, user=self.user_id) - yield self.leave(room=room, user=self.user_id) - - # set [invite/join/left] of self, set [invite/join/left] of other, - # expect all 403s - for usr in [self.user_id, self.rmcreator_id]: - yield self.change_membership( - room=room, - src=self.user_id, - targ=usr, - membership=Membership.INVITE, - expect_code=403 - ) - - yield self.change_membership( - room=room, - src=self.user_id, - targ=usr, - membership=Membership.JOIN, - expect_code=403 - ) - - # It is always valid to LEAVE if you've already left (currently.) - yield self.change_membership( - room=room, - src=self.user_id, - targ=self.rmcreator_id, - membership=Membership.LEAVE, - expect_code=403 - ) - - -class RoomsMemberListTestCase(RestTestCase): - """ Tests /rooms/$room_id/members/list REST events.""" - user_id = "@sid1:red" - - @defer.inlineCallbacks - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "red", - db_pool=db_pool, - http_client=None, - replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, - ) - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - hs.get_handlers().federation_handler = Mock() - - self.auth_user_id = self.user_id - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(self.auth_user_id), - "admin": False, - "device_id": None, - } - hs.get_auth().get_user_by_token = _get_user_by_token - - def _insert_client_ip(*args, **kwargs): - return defer.succeed(None) - hs.get_datastore().insert_client_ip = _insert_client_ip - - synapse.rest.room.register_servlets(hs, self.mock_resource) - - def tearDown(self): - pass - - @defer.inlineCallbacks - def test_get_member_list(self): - room_id = yield self.create_room_as(self.user_id) - (code, response) = yield self.mock_resource.trigger_get( - "/rooms/%s/members" % room_id) - self.assertEquals(200, code, msg=str(response)) - - @defer.inlineCallbacks - def test_get_member_list_no_room(self): - (code, response) = yield self.mock_resource.trigger_get( - "/rooms/roomdoesnotexist/members") - self.assertEquals(403, code, msg=str(response)) - - @defer.inlineCallbacks - def test_get_member_list_no_permission(self): - room_id = yield self.create_room_as("@some_other_guy:red") - (code, response) = yield self.mock_resource.trigger_get( - "/rooms/%s/members" % room_id) - self.assertEquals(403, code, msg=str(response)) - - @defer.inlineCallbacks - def test_get_member_list_mixed_memberships(self): - room_creator = "@some_other_guy:red" - room_id = yield self.create_room_as(room_creator) - room_path = "/rooms/%s/members" % room_id - yield self.invite(room=room_id, src=room_creator, - targ=self.user_id) - # can't see list if you're just invited. - (code, response) = yield self.mock_resource.trigger_get(room_path) - self.assertEquals(403, code, msg=str(response)) - - yield self.join(room=room_id, user=self.user_id) - # can see list now joined - (code, response) = yield self.mock_resource.trigger_get(room_path) - self.assertEquals(200, code, msg=str(response)) - - yield self.leave(room=room_id, user=self.user_id) - # can no longer see list, you've left. - (code, response) = yield self.mock_resource.trigger_get(room_path) - self.assertEquals(403, code, msg=str(response)) - - -class RoomsCreateTestCase(RestTestCase): - """ Tests /rooms and /rooms/$room_id REST events. """ - user_id = "@sid1:red" - - @defer.inlineCallbacks - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.auth_user_id = self.user_id - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "red", - db_pool=db_pool, - http_client=None, - replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, - ) - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - hs.get_handlers().federation_handler = Mock() - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(self.auth_user_id), - "admin": False, - "device_id": None, - } - hs.get_auth().get_user_by_token = _get_user_by_token - - def _insert_client_ip(*args, **kwargs): - return defer.succeed(None) - hs.get_datastore().insert_client_ip = _insert_client_ip - - synapse.rest.room.register_servlets(hs, self.mock_resource) - - def tearDown(self): - pass - - @defer.inlineCallbacks - def test_post_room_no_keys(self): - # POST with no config keys, expect new room id - (code, response) = yield self.mock_resource.trigger("POST", - "/createRoom", - "{}") - self.assertEquals(200, code, response) - self.assertTrue("room_id" in response) - - @defer.inlineCallbacks - def test_post_room_visibility_key(self): - # POST with visibility config key, expect new room id - (code, response) = yield self.mock_resource.trigger( - "POST", - "/createRoom", - '{"visibility":"private"}') - self.assertEquals(200, code) - self.assertTrue("room_id" in response) - - @defer.inlineCallbacks - def test_post_room_custom_key(self): - # POST with custom config keys, expect new room id - (code, response) = yield self.mock_resource.trigger( - "POST", - "/createRoom", - '{"custom":"stuff"}') - self.assertEquals(200, code) - self.assertTrue("room_id" in response) - - @defer.inlineCallbacks - def test_post_room_known_and_unknown_keys(self): - # POST with custom + known config keys, expect new room id - (code, response) = yield self.mock_resource.trigger( - "POST", - "/createRoom", - '{"visibility":"private","custom":"things"}') - self.assertEquals(200, code) - self.assertTrue("room_id" in response) - - @defer.inlineCallbacks - def test_post_room_invalid_content(self): - # POST with invalid content / paths, expect 400 - (code, response) = yield self.mock_resource.trigger( - "POST", - "/createRoom", - '{"visibili') - self.assertEquals(400, code) - - (code, response) = yield self.mock_resource.trigger( - "POST", - "/createRoom", - '["hello"]') - self.assertEquals(400, code) - - -class RoomTopicTestCase(RestTestCase): - """ Tests /rooms/$room_id/topic REST events. """ - user_id = "@sid1:red" - - @defer.inlineCallbacks - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.auth_user_id = self.user_id - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "red", - db_pool=db_pool, - http_client=None, - replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, - ) - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - hs.get_handlers().federation_handler = Mock() - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(self.auth_user_id), - "admin": False, - "device_id": None, - } - - hs.get_auth().get_user_by_token = _get_user_by_token - - def _insert_client_ip(*args, **kwargs): - return defer.succeed(None) - hs.get_datastore().insert_client_ip = _insert_client_ip - - synapse.rest.room.register_servlets(hs, self.mock_resource) - - # create the room - self.room_id = yield self.create_room_as(self.user_id) - self.path = "/rooms/%s/state/m.room.topic" % (self.room_id,) - - def tearDown(self): - pass - - @defer.inlineCallbacks - def test_invalid_puts(self): - # missing keys or invalid json - (code, response) = yield self.mock_resource.trigger("PUT", - self.path, '{}') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - self.path, '{"_name":"bob"}') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - self.path, '{"nao') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - self.path, '[{"_name":"bob"},{"_name":"jill"}]') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - self.path, 'text only') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - self.path, '') - self.assertEquals(400, code, msg=str(response)) - - # valid key, wrong type - content = '{"topic":["Topic name"]}' - (code, response) = yield self.mock_resource.trigger("PUT", - self.path, content) - self.assertEquals(400, code, msg=str(response)) - - @defer.inlineCallbacks - def test_rooms_topic(self): - # nothing should be there - (code, response) = yield self.mock_resource.trigger_get(self.path) - self.assertEquals(404, code, msg=str(response)) - - # valid put - content = '{"topic":"Topic name"}' - (code, response) = yield self.mock_resource.trigger("PUT", - self.path, content) - self.assertEquals(200, code, msg=str(response)) - - # valid get - (code, response) = yield self.mock_resource.trigger_get(self.path) - self.assertEquals(200, code, msg=str(response)) - self.assert_dict(json.loads(content), response) - - @defer.inlineCallbacks - def test_rooms_topic_with_extra_keys(self): - # valid put with extra keys - content = '{"topic":"Seasons","subtopic":"Summer"}' - (code, response) = yield self.mock_resource.trigger("PUT", - self.path, content) - self.assertEquals(200, code, msg=str(response)) - - # valid get - (code, response) = yield self.mock_resource.trigger_get(self.path) - self.assertEquals(200, code, msg=str(response)) - self.assert_dict(json.loads(content), response) - - -class RoomMemberStateTestCase(RestTestCase): - """ Tests /rooms/$room_id/members/$user_id/state REST events. """ - user_id = "@sid1:red" - - @defer.inlineCallbacks - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.auth_user_id = self.user_id - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "red", - db_pool=db_pool, - http_client=None, - replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, - ) - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - hs.get_handlers().federation_handler = Mock() - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(self.auth_user_id), - "admin": False, - "device_id": None, - } - hs.get_auth().get_user_by_token = _get_user_by_token - - def _insert_client_ip(*args, **kwargs): - return defer.succeed(None) - hs.get_datastore().insert_client_ip = _insert_client_ip - - synapse.rest.room.register_servlets(hs, self.mock_resource) - - self.room_id = yield self.create_room_as(self.user_id) - - def tearDown(self): - pass - - @defer.inlineCallbacks - def test_invalid_puts(self): - path = "/rooms/%s/state/m.room.member/%s" % (self.room_id, self.user_id) - # missing keys or invalid json - (code, response) = yield self.mock_resource.trigger("PUT", - path, '{}') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - path, '{"_name":"bob"}') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - path, '{"nao') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - path, '[{"_name":"bob"},{"_name":"jill"}]') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - path, 'text only') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - path, '') - self.assertEquals(400, code, msg=str(response)) - - # valid keys, wrong types - content = ('{"membership":["%s","%s","%s"]}' % - (Membership.INVITE, Membership.JOIN, Membership.LEAVE)) - (code, response) = yield self.mock_resource.trigger("PUT", path, content) - self.assertEquals(400, code, msg=str(response)) - - @defer.inlineCallbacks - def test_rooms_members_self(self): - path = "/rooms/%s/state/m.room.member/%s" % ( - urllib.quote(self.room_id), self.user_id - ) - - # valid join message (NOOP since we made the room) - content = '{"membership":"%s"}' % Membership.JOIN - (code, response) = yield self.mock_resource.trigger("PUT", path, content) - self.assertEquals(200, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("GET", path, None) - self.assertEquals(200, code, msg=str(response)) - - expected_response = { - "membership": Membership.JOIN, - } - self.assertEquals(expected_response, response) - - @defer.inlineCallbacks - def test_rooms_members_other(self): - self.other_id = "@zzsid1:red" - path = "/rooms/%s/state/m.room.member/%s" % ( - urllib.quote(self.room_id), self.other_id - ) - - # valid invite message - content = '{"membership":"%s"}' % Membership.INVITE - (code, response) = yield self.mock_resource.trigger("PUT", path, content) - self.assertEquals(200, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("GET", path, None) - self.assertEquals(200, code, msg=str(response)) - self.assertEquals(json.loads(content), response) - - @defer.inlineCallbacks - def test_rooms_members_other_custom_keys(self): - self.other_id = "@zzsid1:red" - path = "/rooms/%s/state/m.room.member/%s" % ( - urllib.quote(self.room_id), self.other_id - ) - - # valid invite message with custom key - content = ('{"membership":"%s","invite_text":"%s"}' % - (Membership.INVITE, "Join us!")) - (code, response) = yield self.mock_resource.trigger("PUT", path, content) - self.assertEquals(200, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("GET", path, None) - self.assertEquals(200, code, msg=str(response)) - self.assertEquals(json.loads(content), response) - - -class RoomMessagesTestCase(RestTestCase): - """ Tests /rooms/$room_id/messages/$user_id/$msg_id REST events. """ - user_id = "@sid1:red" - - @defer.inlineCallbacks - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.auth_user_id = self.user_id - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "red", - db_pool=db_pool, - http_client=None, - replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, - ) - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - hs.get_handlers().federation_handler = Mock() - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(self.auth_user_id), - "admin": False, - "device_id": None, - } - hs.get_auth().get_user_by_token = _get_user_by_token - - def _insert_client_ip(*args, **kwargs): - return defer.succeed(None) - hs.get_datastore().insert_client_ip = _insert_client_ip - - synapse.rest.room.register_servlets(hs, self.mock_resource) - - self.room_id = yield self.create_room_as(self.user_id) - - def tearDown(self): - pass - - @defer.inlineCallbacks - def test_invalid_puts(self): - path = "/rooms/%s/send/m.room.message/mid1" % ( - urllib.quote(self.room_id)) - # missing keys or invalid json - (code, response) = yield self.mock_resource.trigger("PUT", - path, '{}') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - path, '{"_name":"bob"}') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - path, '{"nao') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - path, '[{"_name":"bob"},{"_name":"jill"}]') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - path, 'text only') - self.assertEquals(400, code, msg=str(response)) - - (code, response) = yield self.mock_resource.trigger("PUT", - path, '') - self.assertEquals(400, code, msg=str(response)) - - @defer.inlineCallbacks - def test_rooms_messages_sent(self): - path = "/rooms/%s/send/m.room.message/mid1" % ( - urllib.quote(self.room_id)) - - content = '{"body":"test","msgtype":{"type":"a"}}' - (code, response) = yield self.mock_resource.trigger("PUT", path, content) - self.assertEquals(400, code, msg=str(response)) - - # custom message types - content = '{"body":"test","msgtype":"test.custom.text"}' - (code, response) = yield self.mock_resource.trigger("PUT", path, content) - self.assertEquals(200, code, msg=str(response)) - -# (code, response) = yield self.mock_resource.trigger("GET", path, None) -# self.assertEquals(200, code, msg=str(response)) -# self.assert_dict(json.loads(content), response) - - # m.text message type - path = "/rooms/%s/send/m.room.message/mid2" % ( - urllib.quote(self.room_id)) - content = '{"body":"test2","msgtype":"m.text"}' - (code, response) = yield self.mock_resource.trigger("PUT", path, content) - self.assertEquals(200, code, msg=str(response)) - - -class RoomInitialSyncTestCase(RestTestCase): - """ Tests /rooms/$room_id/initialSync. """ - user_id = "@sid1:red" - - @defer.inlineCallbacks - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.auth_user_id = self.user_id - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "red", - db_pool=db_pool, - http_client=None, - replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, - ) - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - hs.get_handlers().federation_handler = Mock() - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(self.auth_user_id), - "admin": False, - "device_id": None, - } - hs.get_auth().get_user_by_token = _get_user_by_token - - def _insert_client_ip(*args, **kwargs): - return defer.succeed(None) - hs.get_datastore().insert_client_ip = _insert_client_ip - - synapse.rest.room.register_servlets(hs, self.mock_resource) - - # Since I'm getting my own presence I need to exist as far as presence - # is concerned. - hs.get_handlers().presence_handler.registered_user( - hs.parse_userid(self.user_id) - ) - - # create the room - self.room_id = yield self.create_room_as(self.user_id) - - @defer.inlineCallbacks - def test_initial_sync(self): - (code, response) = yield self.mock_resource.trigger_get( - "/rooms/%s/initialSync" % self.room_id) - self.assertEquals(200, code) - - self.assertEquals(self.room_id, response["room_id"]) - self.assertEquals("join", response["membership"]) - - # Room state is easier to assert on if we unpack it into a dict - state = {} - for event in response["state"]: - if "state_key" not in event: - continue - t = event["type"] - if t not in state: - state[t] = [] - state[t].append(event) - - self.assertTrue("m.room.create" in state) - - self.assertTrue("messages" in response) - self.assertTrue("chunk" in response["messages"]) - self.assertTrue("end" in response["messages"]) - - self.assertTrue("presence" in response) - - presence_by_user = {e["content"]["user_id"]: e - for e in response["presence"] - } - self.assertTrue(self.user_id in presence_by_user) - self.assertEquals("m.presence", presence_by_user[self.user_id]["type"]) diff --git a/tests/rest/test_typing.py b/tests/rest/test_typing.py deleted file mode 100644 index 18138af1b5..0000000000 --- a/tests/rest/test_typing.py +++ /dev/null @@ -1,164 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests REST events for /rooms paths.""" - -# twisted imports -from twisted.internet import defer - -import synapse.rest.room -from synapse.server import HomeServer - -from ..utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey -from .utils import RestTestCase - -from mock import Mock, NonCallableMock - - -PATH_PREFIX = "/_matrix/client/api/v1" - - -class RoomTypingTestCase(RestTestCase): - """ Tests /rooms/$room_id/typing/$user_id REST API. """ - user_id = "@sid:red" - - @defer.inlineCallbacks - def setUp(self): - self.clock = MockClock() - - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.auth_user_id = self.user_id - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "red", - clock=self.clock, - db_pool=db_pool, - http_client=None, - replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, - ) - self.hs = hs - - self.event_source = hs.get_event_sources().sources["typing"] - - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - hs.get_handlers().federation_handler = Mock() - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(self.auth_user_id), - "admin": False, - "device_id": None, - } - - hs.get_auth().get_user_by_token = _get_user_by_token - - def _insert_client_ip(*args, **kwargs): - return defer.succeed(None) - hs.get_datastore().insert_client_ip = _insert_client_ip - - def get_room_members(room_id): - if room_id == self.room_id: - return defer.succeed([hs.parse_userid(self.user_id)]) - else: - return defer.succeed([]) - - @defer.inlineCallbacks - def fetch_room_distributions_into(room_id, localusers=None, - remotedomains=None, ignore_user=None): - - members = yield get_room_members(room_id) - for member in members: - if ignore_user is not None and member == ignore_user: - continue - - if hs.is_mine(member): - if localusers is not None: - localusers.add(member) - else: - if remotedomains is not None: - remotedomains.add(member.domain) - hs.get_handlers().room_member_handler.fetch_room_distributions_into = ( - fetch_room_distributions_into) - - synapse.rest.room.register_servlets(hs, self.mock_resource) - - self.room_id = yield self.create_room_as(self.user_id) - # Need another user to make notifications actually work - yield self.join(self.room_id, user="@jim:red") - - def tearDown(self): - self.hs.get_handlers().typing_notification_handler.tearDown() - - @defer.inlineCallbacks - def test_set_typing(self): - (code, _) = yield self.mock_resource.trigger("PUT", - "/rooms/%s/typing/%s" % (self.room_id, self.user_id), - '{"typing": true, "timeout": 30000}' - ) - self.assertEquals(200, code) - - self.assertEquals(self.event_source.get_current_key(), 1) - self.assertEquals( - self.event_source.get_new_events_for_user(self.user_id, 0, None)[0], - [ - {"type": "m.typing", - "room_id": self.room_id, - "content": { - "user_ids": [self.user_id], - }}, - ] - ) - - @defer.inlineCallbacks - def test_set_not_typing(self): - (code, _) = yield self.mock_resource.trigger("PUT", - "/rooms/%s/typing/%s" % (self.room_id, self.user_id), - '{"typing": false}' - ) - self.assertEquals(200, code) - - @defer.inlineCallbacks - def test_typing_timeout(self): - (code, _) = yield self.mock_resource.trigger("PUT", - "/rooms/%s/typing/%s" % (self.room_id, self.user_id), - '{"typing": true, "timeout": 30000}' - ) - self.assertEquals(200, code) - - self.assertEquals(self.event_source.get_current_key(), 1) - - self.clock.advance_time(31); - - self.assertEquals(self.event_source.get_current_key(), 2) - - (code, _) = yield self.mock_resource.trigger("PUT", - "/rooms/%s/typing/%s" % (self.room_id, self.user_id), - '{"typing": true, "timeout": 30000}' - ) - self.assertEquals(200, code) - - self.assertEquals(self.event_source.get_current_key(), 3) diff --git a/tests/rest/utils.py b/tests/rest/utils.py deleted file mode 100644 index 579441fb4a..0000000000 --- a/tests/rest/utils.py +++ /dev/null @@ -1,134 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# twisted imports -from twisted.internet import defer - -# trial imports -from tests import unittest - -from synapse.api.constants import Membership - -import json -import time - - -class RestTestCase(unittest.TestCase): - """Contains extra helper functions to quickly and clearly perform a given - REST action, which isn't the focus of the test. - - This subclass assumes there are mock_resource and auth_user_id attributes. - """ - - def __init__(self, *args, **kwargs): - super(RestTestCase, self).__init__(*args, **kwargs) - self.mock_resource = None - self.auth_user_id = None - - def mock_get_user_by_token(self, token=None): - return self.auth_user_id - - @defer.inlineCallbacks - def create_room_as(self, room_creator, is_public=True, tok=None): - temp_id = self.auth_user_id - self.auth_user_id = room_creator - path = "/createRoom" - content = "{}" - if not is_public: - content = '{"visibility":"private"}' - if tok: - path = path + "?access_token=%s" % tok - (code, response) = yield self.mock_resource.trigger("POST", path, content) - self.assertEquals(200, code, msg=str(response)) - self.auth_user_id = temp_id - defer.returnValue(response["room_id"]) - - @defer.inlineCallbacks - def invite(self, room=None, src=None, targ=None, expect_code=200, tok=None): - yield self.change_membership(room=room, src=src, targ=targ, tok=tok, - membership=Membership.INVITE, - expect_code=expect_code) - - @defer.inlineCallbacks - def join(self, room=None, user=None, expect_code=200, tok=None): - yield self.change_membership(room=room, src=user, targ=user, tok=tok, - membership=Membership.JOIN, - expect_code=expect_code) - - @defer.inlineCallbacks - def leave(self, room=None, user=None, expect_code=200, tok=None): - yield self.change_membership(room=room, src=user, targ=user, tok=tok, - membership=Membership.LEAVE, - expect_code=expect_code) - - @defer.inlineCallbacks - def change_membership(self, room, src, targ, membership, tok=None, - expect_code=200): - temp_id = self.auth_user_id - self.auth_user_id = src - - path = "/rooms/%s/state/m.room.member/%s" % (room, targ) - if tok: - path = path + "?access_token=%s" % tok - - data = { - "membership": membership - } - - (code, response) = yield self.mock_resource.trigger("PUT", path, - json.dumps(data)) - self.assertEquals(expect_code, code, msg=str(response)) - - self.auth_user_id = temp_id - - @defer.inlineCallbacks - def register(self, user_id): - (code, response) = yield self.mock_resource.trigger( - "POST", - "/register", - json.dumps({ - "user": user_id, - "password": "test", - "type": "m.login.password" - })) - self.assertEquals(200, code) - defer.returnValue(response) - - @defer.inlineCallbacks - def send(self, room_id, body=None, txn_id=None, tok=None, - expect_code=200): - if txn_id is None: - txn_id = "m%s" % (str(time.time())) - if body is None: - body = "body_text_here" - - path = "/rooms/%s/send/m.room.message/%s" % (room_id, txn_id) - content = '{"msgtype":"m.text","body":"%s"}' % body - if tok: - path = path + "?access_token=%s" % tok - - (code, response) = yield self.mock_resource.trigger("PUT", path, content) - self.assertEquals(expect_code, code, msg=str(response)) - - def assert_dict(self, required, actual): - """Does a partial assert of a dict. - - Args: - required (dict): The keys and value which MUST be in 'actual'. - actual (dict): The test result. Extra keys will not be checked. - """ - for key in required: - self.assertEquals(required[key], actual[key], - msg="%s mismatch. %s" % (key, actual)) -- cgit 1.4.1 From b1b85753d759f7127fbb1c4a95005fffd3da7f4d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Jan 2015 15:50:17 +0000 Subject: Add support for storing rejected events in EventContext and data stores --- synapse/events/snapshot.py | 1 + synapse/storage/__init__.py | 11 ++++++++--- synapse/storage/_base.py | 21 +++++++++++++-------- synapse/storage/rejections.py | 33 +++++++++++++++++++++++++++++++++ synapse/storage/schema/delta/v12.sql | 21 +++++++++++++++++++++ synapse/storage/schema/im.sql | 7 +++++++ 6 files changed, 83 insertions(+), 11 deletions(-) create mode 100644 synapse/storage/rejections.py create mode 100644 synapse/storage/schema/delta/v12.sql (limited to 'synapse') diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 6bbba8d6ba..7e98bdef28 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -20,3 +20,4 @@ class EventContext(object): self.current_state = current_state self.auth_events = auth_events self.state_group = None + self.rejected = False diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4beb951b9f..015fcc8775 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,6 +30,7 @@ from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore from .media_repository import MediaRepositoryStore +from .rejections import RejectionsStore from .state import StateStore from .signatures import SignatureStore @@ -66,7 +67,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 11 +SCHEMA_VERSION = 12 class _RollbackButIsFineException(Exception): @@ -82,6 +83,7 @@ class DataStore(RoomMemberStore, RoomStore, DirectoryStore, KeyStore, StateStore, SignatureStore, EventFederationStore, MediaRepositoryStore, + RejectionsStore, ): def __init__(self, hs): @@ -224,6 +226,9 @@ class DataStore(RoomMemberStore, RoomStore, if not outlier: self._store_state_groups_txn(txn, event, context) + if context.rejected: + self._store_rejections_txn(txn, event.event_id, context.rejected) + if current_state: txn.execute( "DELETE FROM current_state_events WHERE room_id = ?", @@ -262,7 +267,7 @@ class DataStore(RoomMemberStore, RoomStore, or_replace=True, ) - if is_new_state: + if is_new_state and not context.rejected: self._simple_insert_txn( txn, "current_state_events", @@ -288,7 +293,7 @@ class DataStore(RoomMemberStore, RoomStore, or_ignore=True, ) - if not backfilled: + if not backfilled and not context.rejected: self._simple_insert_txn( txn, table="state_forward_extremities", diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f660fc6eaf..2075a018b2 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -458,10 +458,12 @@ class SQLBaseStore(object): return [e for e in events if e] def _get_event_txn(self, txn, event_id, check_redacted=True, - get_prev_content=False): + get_prev_content=False, allow_rejected=False): sql = ( - "SELECT internal_metadata, json, r.event_id FROM event_json as e " + "SELECT internal_metadata, json, r.event_id, reason " + "FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " + "LEFT JOIN rejections as rej on rej.event_id = e.event_id " "WHERE e.event_id = ? " "LIMIT 1 " ) @@ -473,13 +475,16 @@ class SQLBaseStore(object): if not res: return None - internal_metadata, js, redacted = res + internal_metadata, js, redacted, rejected_reason = res - return self._get_event_from_row_txn( - txn, internal_metadata, js, redacted, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - ) + if allow_rejected or not rejected_reason: + return self._get_event_from_row_txn( + txn, internal_metadata, js, redacted, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + else: + return None def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False): diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py new file mode 100644 index 0000000000..7d38b31f44 --- /dev/null +++ b/synapse/storage/rejections.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import SQLBaseStore + +import logging + +logger = logging.getLogger(__name__) + + +class RejectionsStore(SQLBaseStore): + def _store_rejections_txn(self, txn, event_id, reason): + self._simple_insert_txn( + txn, + table="rejections", + values={ + "event_id": event_id, + "reason": reason, + "last_failure": self._clock.time_msec(), + } + ) diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql new file mode 100644 index 0000000000..bd2a8b1bb5 --- /dev/null +++ b/synapse/storage/schema/delta/v12.sql @@ -0,0 +1,21 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE +); diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index dd00c1cd2f..bc7c6b6ed5 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -123,3 +123,10 @@ CREATE TABLE IF NOT EXISTS room_hosts( ); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); + +CREATE TABLE IF NOT EXISTS rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE +); -- cgit 1.4.1 From 73dd81ca62885110f7fe0e51e7f4e0183b495d17 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Jan 2015 15:57:08 +0000 Subject: fix pyflakes --- synapse/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 5b622ad3b1..d9fdfb34be 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -295,7 +295,7 @@ class StateHandler(object): # get around circular deps. self.hs.get_auth().check(event, auth_events) return event - except AuthError as e: + except AuthError: pass # Oh dear. -- cgit 1.4.1 From 97c68c508dac6b4b3203b3bc475ffdfd185b6e03 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 22 Jan 2015 16:10:07 +0000 Subject: Move rest APIs back under the rest directory --- synapse/app/homeserver.py | 4 +- synapse/client/__init__.py | 14 - synapse/client/v1/__init__.py | 47 --- synapse/client/v1/admin.py | 47 --- synapse/client/v1/base.py | 80 ---- synapse/client/v1/directory.py | 112 ------ synapse/client/v1/events.py | 81 ---- synapse/client/v1/initial_sync.py | 44 --- synapse/client/v1/login.py | 109 ------ synapse/client/v1/presence.py | 145 -------- synapse/client/v1/profile.py | 113 ------ synapse/client/v1/register.py | 291 --------------- synapse/client/v1/room.py | 559 ---------------------------- synapse/client/v1/transactions.py | 95 ----- synapse/client/v1/voip.py | 60 --- synapse/media/__init__.py | 0 synapse/media/v0/__init__.py | 0 synapse/media/v0/content_repository.py | 212 ----------- synapse/media/v1/__init__.py | 45 --- synapse/media/v1/base_resource.py | 378 ------------------- synapse/media/v1/download_resource.py | 74 ---- synapse/media/v1/filepath.py | 67 ---- synapse/media/v1/media_repository.py | 77 ---- synapse/media/v1/thumbnail_resource.py | 193 ---------- synapse/media/v1/thumbnailer.py | 89 ----- synapse/media/v1/upload_resource.py | 113 ------ synapse/rest/__init__.py | 14 + synapse/rest/client/__init__.py | 14 + synapse/rest/client/v1/__init__.py | 47 +++ synapse/rest/client/v1/admin.py | 47 +++ synapse/rest/client/v1/base.py | 80 ++++ synapse/rest/client/v1/directory.py | 112 ++++++ synapse/rest/client/v1/events.py | 81 ++++ synapse/rest/client/v1/initial_sync.py | 44 +++ synapse/rest/client/v1/login.py | 109 ++++++ synapse/rest/client/v1/presence.py | 145 ++++++++ synapse/rest/client/v1/profile.py | 113 ++++++ synapse/rest/client/v1/register.py | 291 +++++++++++++++ synapse/rest/client/v1/room.py | 559 ++++++++++++++++++++++++++++ synapse/rest/client/v1/transactions.py | 95 +++++ synapse/rest/client/v1/voip.py | 60 +++ synapse/rest/media/__init__.py | 0 synapse/rest/media/v0/__init__.py | 0 synapse/rest/media/v0/content_repository.py | 212 +++++++++++ synapse/rest/media/v1/__init__.py | 45 +++ synapse/rest/media/v1/base_resource.py | 378 +++++++++++++++++++ synapse/rest/media/v1/download_resource.py | 74 ++++ synapse/rest/media/v1/filepath.py | 67 ++++ synapse/rest/media/v1/media_repository.py | 77 ++++ synapse/rest/media/v1/thumbnail_resource.py | 193 ++++++++++ synapse/rest/media/v1/thumbnailer.py | 89 +++++ synapse/rest/media/v1/upload_resource.py | 113 ++++++ synapse/server.py | 2 +- tests/client/v1/test_events.py | 12 +- tests/client/v1/test_rooms.py | 16 +- tests/client/v1/test_typing.py | 4 +- 56 files changed, 3078 insertions(+), 3064 deletions(-) delete mode 100644 synapse/client/__init__.py delete mode 100644 synapse/client/v1/__init__.py delete mode 100644 synapse/client/v1/admin.py delete mode 100644 synapse/client/v1/base.py delete mode 100644 synapse/client/v1/directory.py delete mode 100644 synapse/client/v1/events.py delete mode 100644 synapse/client/v1/initial_sync.py delete mode 100644 synapse/client/v1/login.py delete mode 100644 synapse/client/v1/presence.py delete mode 100644 synapse/client/v1/profile.py delete mode 100644 synapse/client/v1/register.py delete mode 100644 synapse/client/v1/room.py delete mode 100644 synapse/client/v1/transactions.py delete mode 100644 synapse/client/v1/voip.py delete mode 100644 synapse/media/__init__.py delete mode 100644 synapse/media/v0/__init__.py delete mode 100644 synapse/media/v0/content_repository.py delete mode 100644 synapse/media/v1/__init__.py delete mode 100644 synapse/media/v1/base_resource.py delete mode 100644 synapse/media/v1/download_resource.py delete mode 100644 synapse/media/v1/filepath.py delete mode 100644 synapse/media/v1/media_repository.py delete mode 100644 synapse/media/v1/thumbnail_resource.py delete mode 100644 synapse/media/v1/thumbnailer.py delete mode 100644 synapse/media/v1/upload_resource.py create mode 100644 synapse/rest/__init__.py create mode 100644 synapse/rest/client/__init__.py create mode 100644 synapse/rest/client/v1/__init__.py create mode 100644 synapse/rest/client/v1/admin.py create mode 100644 synapse/rest/client/v1/base.py create mode 100644 synapse/rest/client/v1/directory.py create mode 100644 synapse/rest/client/v1/events.py create mode 100644 synapse/rest/client/v1/initial_sync.py create mode 100644 synapse/rest/client/v1/login.py create mode 100644 synapse/rest/client/v1/presence.py create mode 100644 synapse/rest/client/v1/profile.py create mode 100644 synapse/rest/client/v1/register.py create mode 100644 synapse/rest/client/v1/room.py create mode 100644 synapse/rest/client/v1/transactions.py create mode 100644 synapse/rest/client/v1/voip.py create mode 100644 synapse/rest/media/__init__.py create mode 100644 synapse/rest/media/v0/__init__.py create mode 100644 synapse/rest/media/v0/content_repository.py create mode 100644 synapse/rest/media/v1/__init__.py create mode 100644 synapse/rest/media/v1/base_resource.py create mode 100644 synapse/rest/media/v1/download_resource.py create mode 100644 synapse/rest/media/v1/filepath.py create mode 100644 synapse/rest/media/v1/media_repository.py create mode 100644 synapse/rest/media/v1/thumbnail_resource.py create mode 100644 synapse/rest/media/v1/thumbnailer.py create mode 100644 synapse/rest/media/v1/upload_resource.py (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index afe3d19760..cd24bbdc79 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,8 +26,8 @@ from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect -from synapse.media.v0.content_repository import ContentRepoResource -from synapse.media.v1.media_repository import MediaRepositoryResource +from synapse.rest.media.v0.content_repository import ContentRepoResource +from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( diff --git a/synapse/client/__init__.py b/synapse/client/__init__.py deleted file mode 100644 index 1a84d94cd9..0000000000 --- a/synapse/client/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/synapse/client/v1/__init__.py b/synapse/client/v1/__init__.py deleted file mode 100644 index 88ec9cd27d..0000000000 --- a/synapse/client/v1/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from . import ( - room, events, register, login, profile, presence, initial_sync, directory, - voip, admin, -) - - -class RestServletFactory(object): - - """ A factory for creating REST servlets. - - These REST servlets represent the entire client-server REST API. Generally - speaking, they serve as wrappers around events and the handlers that - process them. - - See synapse.events for information on synapse events. - """ - - def __init__(self, hs): - client_resource = hs.get_resource_for_client() - - # TODO(erikj): There *must* be a better way of doing this. - room.register_servlets(hs, client_resource) - events.register_servlets(hs, client_resource) - register.register_servlets(hs, client_resource) - login.register_servlets(hs, client_resource) - profile.register_servlets(hs, client_resource) - presence.register_servlets(hs, client_resource) - initial_sync.register_servlets(hs, client_resource) - directory.register_servlets(hs, client_resource) - voip.register_servlets(hs, client_resource) - admin.register_servlets(hs, client_resource) diff --git a/synapse/client/v1/admin.py b/synapse/client/v1/admin.py deleted file mode 100644 index 0aa83514c8..0000000000 --- a/synapse/client/v1/admin.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from synapse.api.errors import AuthError, SynapseError -from base import RestServlet, client_path_pattern - -import logging - -logger = logging.getLogger(__name__) - - -class WhoisRestServlet(RestServlet): - PATTERN = client_path_pattern("/admin/whois/(?P[^/]*)") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - target_user = self.hs.parse_userid(user_id) - auth_user = yield self.auth.get_user_by_req(request) - is_admin = yield self.auth.is_server_admin(auth_user) - - if not is_admin and target_user != auth_user: - raise AuthError(403, "You are not a server admin") - - if not self.hs.is_mine(target_user): - raise SynapseError(400, "Can only whois a local user") - - ret = yield self.handlers.admin_handler.get_whois(target_user) - - defer.returnValue((200, ret)) - - -def register_servlets(hs, http_server): - WhoisRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/base.py b/synapse/client/v1/base.py deleted file mode 100644 index d005206b77..0000000000 --- a/synapse/client/v1/base.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" This module contains base REST classes for constructing REST servlets. """ -from synapse.api.urls import CLIENT_PREFIX -from .transactions import HttpTransactionStore -import re - -import logging - - -logger = logging.getLogger(__name__) - - -def client_path_pattern(path_regex): - """Creates a regex compiled client path with the correct client path - prefix. - - Args: - path_regex (str): The regex string to match. This should NOT have a ^ - as this will be prefixed. - Returns: - SRE_Pattern - """ - return re.compile("^" + CLIENT_PREFIX + path_regex) - - -class RestServlet(object): - - """ A Synapse REST Servlet. - - An implementing class can either provide its own custom 'register' method, - or use the automatic pattern handling provided by the base class. - - To use this latter, the implementing class instead provides a `PATTERN` - class attribute containing a pre-compiled regular expression. The automatic - register method will then use this method to register any of the following - instance methods associated with the corresponding HTTP method: - - on_GET - on_PUT - on_POST - on_DELETE - on_OPTIONS - - Automatically handles turning CodeMessageExceptions thrown by these methods - into the appropriate HTTP response. - """ - - def __init__(self, hs): - self.hs = hs - - self.handlers = hs.get_handlers() - self.builder_factory = hs.get_event_builder_factory() - self.auth = hs.get_auth() - self.txns = HttpTransactionStore() - - def register(self, http_server): - """ Register this servlet with the given HTTP server. """ - if hasattr(self, "PATTERN"): - pattern = self.PATTERN - - for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"): - if hasattr(self, "on_%s" % (method)): - method_handler = getattr(self, "on_%s" % (method)) - http_server.register_path(method, pattern, method_handler) - else: - raise NotImplementedError("RestServlet must register something.") diff --git a/synapse/client/v1/directory.py b/synapse/client/v1/directory.py deleted file mode 100644 index 7ff44fdd9e..0000000000 --- a/synapse/client/v1/directory.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from twisted.internet import defer - -from synapse.api.errors import AuthError, SynapseError, Codes -from base import RestServlet, client_path_pattern - -import json -import logging - - -logger = logging.getLogger(__name__) - - -def register_servlets(hs, http_server): - ClientDirectoryServer(hs).register(http_server) - - -class ClientDirectoryServer(RestServlet): - PATTERN = client_path_pattern("/directory/room/(?P[^/]*)$") - - @defer.inlineCallbacks - def on_GET(self, request, room_alias): - room_alias = self.hs.parse_roomalias(room_alias) - - dir_handler = self.handlers.directory_handler - res = yield dir_handler.get_association(room_alias) - - defer.returnValue((200, res)) - - @defer.inlineCallbacks - def on_PUT(self, request, room_alias): - user = yield self.auth.get_user_by_req(request) - - content = _parse_json(request) - if not "room_id" in content: - raise SynapseError(400, "Missing room_id key", - errcode=Codes.BAD_JSON) - - logger.debug("Got content: %s", content) - - room_alias = self.hs.parse_roomalias(room_alias) - - logger.debug("Got room name: %s", room_alias.to_string()) - - room_id = content["room_id"] - servers = content["servers"] if "servers" in content else None - - logger.debug("Got room_id: %s", room_id) - logger.debug("Got servers: %s", servers) - - # TODO(erikj): Check types. - # TODO(erikj): Check that room exists - - dir_handler = self.handlers.directory_handler - - try: - user_id = user.to_string() - yield dir_handler.create_association( - user_id, room_alias, room_id, servers - ) - yield dir_handler.send_room_alias_update_event(user_id, room_id) - except SynapseError as e: - raise e - except: - logger.exception("Failed to create association") - raise - - defer.returnValue((200, {})) - - @defer.inlineCallbacks - def on_DELETE(self, request, room_alias): - user = yield self.auth.get_user_by_req(request) - - is_admin = yield self.auth.is_server_admin(user) - if not is_admin: - raise AuthError(403, "You need to be a server admin") - - dir_handler = self.handlers.directory_handler - - room_alias = self.hs.parse_roomalias(room_alias) - - yield dir_handler.delete_association( - user.to_string(), room_alias - ) - - defer.returnValue((200, {})) - - -def _parse_json(request): - try: - content = json.loads(request.content.read()) - if type(content) != dict: - raise SynapseError(400, "Content must be a JSON object.", - errcode=Codes.NOT_JSON) - return content - except ValueError: - raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) diff --git a/synapse/client/v1/events.py b/synapse/client/v1/events.py deleted file mode 100644 index c2515528ac..0000000000 --- a/synapse/client/v1/events.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module contains REST servlets to do with event streaming, /events.""" -from twisted.internet import defer - -from synapse.api.errors import SynapseError -from synapse.streams.config import PaginationConfig -from .base import RestServlet, client_path_pattern - -import logging - - -logger = logging.getLogger(__name__) - - -class EventStreamRestServlet(RestServlet): - PATTERN = client_path_pattern("/events$") - - DEFAULT_LONGPOLL_TIME_MS = 30000 - - @defer.inlineCallbacks - def on_GET(self, request): - auth_user = yield self.auth.get_user_by_req(request) - try: - handler = self.handlers.event_stream_handler - pagin_config = PaginationConfig.from_request(request) - timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS - if "timeout" in request.args: - try: - timeout = int(request.args["timeout"][0]) - except ValueError: - raise SynapseError(400, "timeout must be in milliseconds.") - - as_client_event = "raw" not in request.args - - chunk = yield handler.get_stream( - auth_user.to_string(), pagin_config, timeout=timeout, - as_client_event=as_client_event - ) - except: - logger.exception("Event stream failed") - raise - - defer.returnValue((200, chunk)) - - def on_OPTIONS(self, request): - return (200, {}) - - -# TODO: Unit test gets, with and without auth, with different kinds of events. -class EventRestServlet(RestServlet): - PATTERN = client_path_pattern("/events/(?P[^/]*)$") - - @defer.inlineCallbacks - def on_GET(self, request, event_id): - auth_user = yield self.auth.get_user_by_req(request) - handler = self.handlers.event_handler - event = yield handler.get_event(auth_user, event_id) - - if event: - defer.returnValue((200, self.hs.serialize_event(event))) - else: - defer.returnValue((404, "Event not found.")) - - -def register_servlets(hs, http_server): - EventStreamRestServlet(hs).register(http_server) - EventRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/initial_sync.py b/synapse/client/v1/initial_sync.py deleted file mode 100644 index b13d56b286..0000000000 --- a/synapse/client/v1/initial_sync.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from synapse.streams.config import PaginationConfig -from base import RestServlet, client_path_pattern - - -# TODO: Needs unit testing -class InitialSyncRestServlet(RestServlet): - PATTERN = client_path_pattern("/initialSync$") - - @defer.inlineCallbacks - def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) - with_feedback = "feedback" in request.args - as_client_event = "raw" not in request.args - pagination_config = PaginationConfig.from_request(request) - handler = self.handlers.message_handler - content = yield handler.snapshot_all_rooms( - user_id=user.to_string(), - pagin_config=pagination_config, - feedback=with_feedback, - as_client_event=as_client_event - ) - - defer.returnValue((200, content)) - - -def register_servlets(hs, http_server): - InitialSyncRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/login.py b/synapse/client/v1/login.py deleted file mode 100644 index 6b8deff67b..0000000000 --- a/synapse/client/v1/login.py +++ /dev/null @@ -1,109 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from synapse.api.errors import SynapseError -from synapse.types import UserID -from base import RestServlet, client_path_pattern - -import json - - -class LoginRestServlet(RestServlet): - PATTERN = client_path_pattern("/login$") - PASS_TYPE = "m.login.password" - - def on_GET(self, request): - return (200, {"flows": [{"type": LoginRestServlet.PASS_TYPE}]}) - - def on_OPTIONS(self, request): - return (200, {}) - - @defer.inlineCallbacks - def on_POST(self, request): - login_submission = _parse_json(request) - try: - if login_submission["type"] == LoginRestServlet.PASS_TYPE: - result = yield self.do_password_login(login_submission) - defer.returnValue(result) - else: - raise SynapseError(400, "Bad login type.") - except KeyError: - raise SynapseError(400, "Missing JSON keys.") - - @defer.inlineCallbacks - def do_password_login(self, login_submission): - if not login_submission["user"].startswith('@'): - login_submission["user"] = UserID.create( - login_submission["user"], self.hs.hostname).to_string() - - handler = self.handlers.login_handler - token = yield handler.login( - user=login_submission["user"], - password=login_submission["password"]) - - result = { - "user_id": login_submission["user"], # may have changed - "access_token": token, - "home_server": self.hs.hostname, - } - - defer.returnValue((200, result)) - - -class LoginFallbackRestServlet(RestServlet): - PATTERN = client_path_pattern("/login/fallback$") - - def on_GET(self, request): - # TODO(kegan): This should be returning some HTML which is capable of - # hitting LoginRestServlet - return (200, {}) - - -class PasswordResetRestServlet(RestServlet): - PATTERN = client_path_pattern("/login/reset") - - @defer.inlineCallbacks - def on_POST(self, request): - reset_info = _parse_json(request) - try: - email = reset_info["email"] - user_id = reset_info["user_id"] - handler = self.handlers.login_handler - yield handler.reset_password(user_id, email) - # purposefully give no feedback to avoid people hammering different - # combinations. - defer.returnValue((200, {})) - except KeyError: - raise SynapseError( - 400, - "Missing keys. Requires 'email' and 'user_id'." - ) - - -def _parse_json(request): - try: - content = json.loads(request.content.read()) - if type(content) != dict: - raise SynapseError(400, "Content must be a JSON object.") - return content - except ValueError: - raise SynapseError(400, "Content not JSON.") - - -def register_servlets(hs, http_server): - LoginRestServlet(hs).register(http_server) - # TODO PasswordResetRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/presence.py b/synapse/client/v1/presence.py deleted file mode 100644 index ca4d2d21f0..0000000000 --- a/synapse/client/v1/presence.py +++ /dev/null @@ -1,145 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" This module contains REST servlets to do with presence: /presence/ -""" -from twisted.internet import defer - -from synapse.api.errors import SynapseError -from base import RestServlet, client_path_pattern - -import json -import logging - -logger = logging.getLogger(__name__) - - -class PresenceStatusRestServlet(RestServlet): - PATTERN = client_path_pattern("/presence/(?P[^/]*)/status") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - state = yield self.handlers.presence_handler.get_state( - target_user=user, auth_user=auth_user) - - defer.returnValue((200, state)) - - @defer.inlineCallbacks - def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - state = {} - try: - content = json.loads(request.content.read()) - - state["presence"] = content.pop("presence") - - if "status_msg" in content: - state["status_msg"] = content.pop("status_msg") - if not isinstance(state["status_msg"], basestring): - raise SynapseError(400, "status_msg must be a string.") - - if content: - raise KeyError() - except SynapseError as e: - raise e - except: - raise SynapseError(400, "Unable to parse state") - - yield self.handlers.presence_handler.set_state( - target_user=user, auth_user=auth_user, state=state) - - defer.returnValue((200, {})) - - def on_OPTIONS(self, request): - return (200, {}) - - -class PresenceListRestServlet(RestServlet): - PATTERN = client_path_pattern("/presence/list/(?P[^/]*)") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - if not self.hs.is_mine(user): - raise SynapseError(400, "User not hosted on this Home Server") - - if auth_user != user: - raise SynapseError(400, "Cannot get another user's presence list") - - presence = yield self.handlers.presence_handler.get_presence_list( - observer_user=user, accepted=True) - - for p in presence: - observed_user = p.pop("observed_user") - p["user_id"] = observed_user.to_string() - - defer.returnValue((200, presence)) - - @defer.inlineCallbacks - def on_POST(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - if not self.hs.is_mine(user): - raise SynapseError(400, "User not hosted on this Home Server") - - if auth_user != user: - raise SynapseError( - 400, "Cannot modify another user's presence list") - - try: - content = json.loads(request.content.read()) - except: - logger.exception("JSON parse error") - raise SynapseError(400, "Unable to parse content") - - if "invite" in content: - for u in content["invite"]: - if not isinstance(u, basestring): - raise SynapseError(400, "Bad invite value.") - if len(u) == 0: - continue - invited_user = self.hs.parse_userid(u) - yield self.handlers.presence_handler.send_invite( - observer_user=user, observed_user=invited_user - ) - - if "drop" in content: - for u in content["drop"]: - if not isinstance(u, basestring): - raise SynapseError(400, "Bad drop value.") - if len(u) == 0: - continue - dropped_user = self.hs.parse_userid(u) - yield self.handlers.presence_handler.drop( - observer_user=user, observed_user=dropped_user - ) - - defer.returnValue((200, {})) - - def on_OPTIONS(self, request): - return (200, {}) - - -def register_servlets(hs, http_server): - PresenceStatusRestServlet(hs).register(http_server) - PresenceListRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/profile.py b/synapse/client/v1/profile.py deleted file mode 100644 index dc6eb424b0..0000000000 --- a/synapse/client/v1/profile.py +++ /dev/null @@ -1,113 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" This module contains REST servlets to do with profile: /profile/ """ -from twisted.internet import defer - -from base import RestServlet, client_path_pattern - -import json - - -class ProfileDisplaynameRestServlet(RestServlet): - PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) - - displayname = yield self.handlers.profile_handler.get_displayname( - user, - ) - - defer.returnValue((200, {"displayname": displayname})) - - @defer.inlineCallbacks - def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - try: - content = json.loads(request.content.read()) - new_name = content["displayname"] - except: - defer.returnValue((400, "Unable to parse name")) - - yield self.handlers.profile_handler.set_displayname( - user, auth_user, new_name) - - defer.returnValue((200, {})) - - def on_OPTIONS(self, request, user_id): - return (200, {}) - - -class ProfileAvatarURLRestServlet(RestServlet): - PATTERN = client_path_pattern("/profile/(?P[^/]*)/avatar_url") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) - - avatar_url = yield self.handlers.profile_handler.get_avatar_url( - user, - ) - - defer.returnValue((200, {"avatar_url": avatar_url})) - - @defer.inlineCallbacks - def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) - - try: - content = json.loads(request.content.read()) - new_name = content["avatar_url"] - except: - defer.returnValue((400, "Unable to parse name")) - - yield self.handlers.profile_handler.set_avatar_url( - user, auth_user, new_name) - - defer.returnValue((200, {})) - - def on_OPTIONS(self, request, user_id): - return (200, {}) - - -class ProfileRestServlet(RestServlet): - PATTERN = client_path_pattern("/profile/(?P[^/]*)") - - @defer.inlineCallbacks - def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) - - displayname = yield self.handlers.profile_handler.get_displayname( - user, - ) - avatar_url = yield self.handlers.profile_handler.get_avatar_url( - user, - ) - - defer.returnValue((200, { - "displayname": displayname, - "avatar_url": avatar_url - })) - - -def register_servlets(hs, http_server): - ProfileDisplaynameRestServlet(hs).register(http_server) - ProfileAvatarURLRestServlet(hs).register(http_server) - ProfileRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/register.py b/synapse/client/v1/register.py deleted file mode 100644 index e3b26902d9..0000000000 --- a/synapse/client/v1/register.py +++ /dev/null @@ -1,291 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module contains REST servlets to do with registration: /register""" -from twisted.internet import defer - -from synapse.api.errors import SynapseError, Codes -from synapse.api.constants import LoginType -from base import RestServlet, client_path_pattern -import synapse.util.stringutils as stringutils - -from synapse.util.async import run_on_reactor - -from hashlib import sha1 -import hmac -import json -import logging -import urllib - -logger = logging.getLogger(__name__) - - -# We ought to be using hmac.compare_digest() but on older pythons it doesn't -# exist. It's a _really minor_ security flaw to use plain string comparison -# because the timing attack is so obscured by all the other code here it's -# unlikely to make much difference -if hasattr(hmac, "compare_digest"): - compare_digest = hmac.compare_digest -else: - compare_digest = lambda a, b: a == b - - -class RegisterRestServlet(RestServlet): - """Handles registration with the home server. - - This servlet is in control of the registration flow; the registration - handler doesn't have a concept of multi-stages or sessions. - """ - - PATTERN = client_path_pattern("/register$") - - def __init__(self, hs): - super(RegisterRestServlet, self).__init__(hs) - # sessions are stored as: - # self.sessions = { - # "session_id" : { __session_dict__ } - # } - # TODO: persistent storage - self.sessions = {} - - def on_GET(self, request): - if self.hs.config.enable_registration_captcha: - return ( - 200, - {"flows": [ - { - "type": LoginType.RECAPTCHA, - "stages": [ - LoginType.RECAPTCHA, - LoginType.EMAIL_IDENTITY, - LoginType.PASSWORD - ] - }, - { - "type": LoginType.RECAPTCHA, - "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD] - } - ]} - ) - else: - return ( - 200, - {"flows": [ - { - "type": LoginType.EMAIL_IDENTITY, - "stages": [ - LoginType.EMAIL_IDENTITY, LoginType.PASSWORD - ] - }, - { - "type": LoginType.PASSWORD - } - ]} - ) - - @defer.inlineCallbacks - def on_POST(self, request): - register_json = _parse_json(request) - - session = (register_json["session"] - if "session" in register_json else None) - login_type = None - if "type" not in register_json: - raise SynapseError(400, "Missing 'type' key.") - - try: - login_type = register_json["type"] - stages = { - LoginType.RECAPTCHA: self._do_recaptcha, - LoginType.PASSWORD: self._do_password, - LoginType.EMAIL_IDENTITY: self._do_email_identity - } - - session_info = self._get_session_info(request, session) - logger.debug("%s : session info %s request info %s", - login_type, session_info, register_json) - response = yield stages[login_type]( - request, - register_json, - session_info - ) - - if "access_token" not in response: - # isn't a final response - response["session"] = session_info["id"] - - defer.returnValue((200, response)) - except KeyError as e: - logger.exception(e) - raise SynapseError(400, "Missing JSON keys for login type %s." % ( - login_type, - )) - - def on_OPTIONS(self, request): - return (200, {}) - - def _get_session_info(self, request, session_id): - if not session_id: - # create a new session - while session_id is None or session_id in self.sessions: - session_id = stringutils.random_string(24) - self.sessions[session_id] = { - "id": session_id, - LoginType.EMAIL_IDENTITY: False, - LoginType.RECAPTCHA: False - } - - return self.sessions[session_id] - - def _save_session(self, session): - # TODO: Persistent storage - logger.debug("Saving session %s", session) - self.sessions[session["id"]] = session - - def _remove_session(self, session): - logger.debug("Removing session %s", session) - self.sessions.pop(session["id"]) - - @defer.inlineCallbacks - def _do_recaptcha(self, request, register_json, session): - if not self.hs.config.enable_registration_captcha: - raise SynapseError(400, "Captcha not required.") - - yield self._check_recaptcha(request, register_json, session) - - session[LoginType.RECAPTCHA] = True # mark captcha as done - self._save_session(session) - defer.returnValue({ - "next": [LoginType.PASSWORD, LoginType.EMAIL_IDENTITY] - }) - - @defer.inlineCallbacks - def _check_recaptcha(self, request, register_json, session): - if ("captcha_bypass_hmac" in register_json and - self.hs.config.captcha_bypass_secret): - if "user" not in register_json: - raise SynapseError(400, "Captcha bypass needs 'user'") - - want = hmac.new( - key=self.hs.config.captcha_bypass_secret, - msg=register_json["user"], - digestmod=sha1, - ).hexdigest() - - # str() because otherwise hmac complains that 'unicode' does not - # have the buffer interface - got = str(register_json["captcha_bypass_hmac"]) - - if compare_digest(want, got): - session["user"] = register_json["user"] - defer.returnValue(None) - else: - raise SynapseError( - 400, "Captcha bypass HMAC incorrect", - errcode=Codes.CAPTCHA_NEEDED - ) - - challenge = None - user_response = None - try: - challenge = register_json["challenge"] - user_response = register_json["response"] - except KeyError: - raise SynapseError(400, "Captcha response is required", - errcode=Codes.CAPTCHA_NEEDED) - - ip_addr = self.hs.get_ip_from_request(request) - - handler = self.handlers.registration_handler - yield handler.check_recaptcha( - ip_addr, - self.hs.config.recaptcha_private_key, - challenge, - user_response - ) - - @defer.inlineCallbacks - def _do_email_identity(self, request, register_json, session): - if (self.hs.config.enable_registration_captcha and - not session[LoginType.RECAPTCHA]): - raise SynapseError(400, "Captcha is required.") - - threepidCreds = register_json['threepidCreds'] - handler = self.handlers.registration_handler - logger.debug("Registering email. threepidcreds: %s" % (threepidCreds)) - yield handler.register_email(threepidCreds) - session["threepidCreds"] = threepidCreds # store creds for next stage - session[LoginType.EMAIL_IDENTITY] = True # mark email as done - self._save_session(session) - defer.returnValue({ - "next": LoginType.PASSWORD - }) - - @defer.inlineCallbacks - def _do_password(self, request, register_json, session): - yield run_on_reactor() - if (self.hs.config.enable_registration_captcha and - not session[LoginType.RECAPTCHA]): - # captcha should've been done by this stage! - raise SynapseError(400, "Captcha is required.") - - if ("user" in session and "user" in register_json and - session["user"] != register_json["user"]): - raise SynapseError( - 400, "Cannot change user ID during registration" - ) - - password = register_json["password"].encode("utf-8") - desired_user_id = (register_json["user"].encode("utf-8") - if "user" in register_json else None) - if (desired_user_id - and urllib.quote(desired_user_id) != desired_user_id): - raise SynapseError( - 400, - "User ID must only contain characters which do not " + - "require URL encoding.") - handler = self.handlers.registration_handler - (user_id, token) = yield handler.register( - localpart=desired_user_id, - password=password - ) - - if session[LoginType.EMAIL_IDENTITY]: - logger.debug("Binding emails %s to %s" % ( - session["threepidCreds"], user_id) - ) - yield handler.bind_emails(user_id, session["threepidCreds"]) - - result = { - "user_id": user_id, - "access_token": token, - "home_server": self.hs.hostname, - } - self._remove_session(session) - defer.returnValue(result) - - -def _parse_json(request): - try: - content = json.loads(request.content.read()) - if type(content) != dict: - raise SynapseError(400, "Content must be a JSON object.") - return content - except ValueError: - raise SynapseError(400, "Content not JSON.") - - -def register_servlets(hs, http_server): - RegisterRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/room.py b/synapse/client/v1/room.py deleted file mode 100644 index 48bba2a5f3..0000000000 --- a/synapse/client/v1/room.py +++ /dev/null @@ -1,559 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" This module contains REST servlets to do with rooms: /rooms/ """ -from twisted.internet import defer - -from base import RestServlet, client_path_pattern -from synapse.api.errors import SynapseError, Codes -from synapse.streams.config import PaginationConfig -from synapse.api.constants import EventTypes, Membership - -import json -import logging -import urllib - - -logger = logging.getLogger(__name__) - - -class RoomCreateRestServlet(RestServlet): - # No PATTERN; we have custom dispatch rules here - - def register(self, http_server): - PATTERN = "/createRoom" - register_txn_path(self, PATTERN, http_server) - # define CORS for all of /rooms in RoomCreateRestServlet for simplicity - http_server.register_path("OPTIONS", - client_path_pattern("/rooms(?:/.*)?$"), - self.on_OPTIONS) - # define CORS for /createRoom[/txnid] - http_server.register_path("OPTIONS", - client_path_pattern("/createRoom(?:/.*)?$"), - self.on_OPTIONS) - - @defer.inlineCallbacks - def on_PUT(self, request, txn_id): - try: - defer.returnValue( - self.txns.get_client_transaction(request, txn_id) - ) - except KeyError: - pass - - response = yield self.on_POST(request) - - self.txns.store_client_transaction(request, txn_id, response) - defer.returnValue(response) - - @defer.inlineCallbacks - def on_POST(self, request): - auth_user = yield self.auth.get_user_by_req(request) - - room_config = self.get_room_config(request) - info = yield self.make_room(room_config, auth_user, None) - room_config.update(info) - defer.returnValue((200, info)) - - @defer.inlineCallbacks - def make_room(self, room_config, auth_user, room_id): - handler = self.handlers.room_creation_handler - info = yield handler.create_room( - user_id=auth_user.to_string(), - room_id=room_id, - config=room_config - ) - defer.returnValue(info) - - def get_room_config(self, request): - try: - user_supplied_config = json.loads(request.content.read()) - if "visibility" not in user_supplied_config: - # default visibility - user_supplied_config["visibility"] = "public" - return user_supplied_config - except (ValueError, TypeError): - raise SynapseError(400, "Body must be JSON.", - errcode=Codes.BAD_JSON) - - def on_OPTIONS(self, request): - return (200, {}) - - -# TODO: Needs unit testing for generic events -class RoomStateEventRestServlet(RestServlet): - def register(self, http_server): - # /room/$roomid/state/$eventtype - no_state_key = "/rooms/(?P[^/]*)/state/(?P[^/]*)$" - - # /room/$roomid/state/$eventtype/$statekey - state_key = ("/rooms/(?P[^/]*)/state/" - "(?P[^/]*)/(?P[^/]*)$") - - http_server.register_path("GET", - client_path_pattern(state_key), - self.on_GET) - http_server.register_path("PUT", - client_path_pattern(state_key), - self.on_PUT) - http_server.register_path("GET", - client_path_pattern(no_state_key), - self.on_GET_no_state_key) - http_server.register_path("PUT", - client_path_pattern(no_state_key), - self.on_PUT_no_state_key) - - def on_GET_no_state_key(self, request, room_id, event_type): - return self.on_GET(request, room_id, event_type, "") - - def on_PUT_no_state_key(self, request, room_id, event_type): - return self.on_PUT(request, room_id, event_type, "") - - @defer.inlineCallbacks - def on_GET(self, request, room_id, event_type, state_key): - user = yield self.auth.get_user_by_req(request) - - msg_handler = self.handlers.message_handler - data = yield msg_handler.get_room_data( - user_id=user.to_string(), - room_id=room_id, - event_type=event_type, - state_key=state_key, - ) - - if not data: - raise SynapseError( - 404, "Event not found.", errcode=Codes.NOT_FOUND - ) - defer.returnValue((200, data.get_dict()["content"])) - - @defer.inlineCallbacks - def on_PUT(self, request, room_id, event_type, state_key): - user = yield self.auth.get_user_by_req(request) - - content = _parse_json(request) - - event_dict = { - "type": event_type, - "content": content, - "room_id": room_id, - "sender": user.to_string(), - } - - if state_key is not None: - event_dict["state_key"] = state_key - - msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event(event_dict) - - defer.returnValue((200, {})) - - -# TODO: Needs unit testing for generic events + feedback -class RoomSendEventRestServlet(RestServlet): - - def register(self, http_server): - # /rooms/$roomid/send/$event_type[/$txn_id] - PATTERN = ("/rooms/(?P[^/]*)/send/(?P[^/]*)") - register_txn_path(self, PATTERN, http_server, with_get=True) - - @defer.inlineCallbacks - def on_POST(self, request, room_id, event_type): - user = yield self.auth.get_user_by_req(request) - content = _parse_json(request) - - msg_handler = self.handlers.message_handler - event = yield msg_handler.create_and_send_event( - { - "type": event_type, - "content": content, - "room_id": room_id, - "sender": user.to_string(), - } - ) - - defer.returnValue((200, {"event_id": event.event_id})) - - def on_GET(self, request, room_id, event_type, txn_id): - return (200, "Not implemented") - - @defer.inlineCallbacks - def on_PUT(self, request, room_id, event_type, txn_id): - try: - defer.returnValue( - self.txns.get_client_transaction(request, txn_id) - ) - except KeyError: - pass - - response = yield self.on_POST(request, room_id, event_type) - - self.txns.store_client_transaction(request, txn_id, response) - defer.returnValue(response) - - -# TODO: Needs unit testing for room ID + alias joins -class JoinRoomAliasServlet(RestServlet): - - def register(self, http_server): - # /join/$room_identifier[/$txn_id] - PATTERN = ("/join/(?P[^/]*)") - register_txn_path(self, PATTERN, http_server) - - @defer.inlineCallbacks - def on_POST(self, request, room_identifier): - user = yield self.auth.get_user_by_req(request) - - # the identifier could be a room alias or a room id. Try one then the - # other if it fails to parse, without swallowing other valid - # SynapseErrors. - - identifier = None - is_room_alias = False - try: - identifier = self.hs.parse_roomalias(room_identifier) - is_room_alias = True - except SynapseError: - identifier = self.hs.parse_roomid(room_identifier) - - # TODO: Support for specifying the home server to join with? - - if is_room_alias: - handler = self.handlers.room_member_handler - ret_dict = yield handler.join_room_alias(user, identifier) - defer.returnValue((200, ret_dict)) - else: # room id - msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event( - { - "type": EventTypes.Member, - "content": {"membership": Membership.JOIN}, - "room_id": identifier.to_string(), - "sender": user.to_string(), - "state_key": user.to_string(), - } - ) - - defer.returnValue((200, {"room_id": identifier.to_string()})) - - @defer.inlineCallbacks - def on_PUT(self, request, room_identifier, txn_id): - try: - defer.returnValue( - self.txns.get_client_transaction(request, txn_id) - ) - except KeyError: - pass - - response = yield self.on_POST(request, room_identifier) - - self.txns.store_client_transaction(request, txn_id, response) - defer.returnValue(response) - - -# TODO: Needs unit testing -class PublicRoomListRestServlet(RestServlet): - PATTERN = client_path_pattern("/publicRooms$") - - @defer.inlineCallbacks - def on_GET(self, request): - handler = self.handlers.room_list_handler - data = yield handler.get_public_room_list() - defer.returnValue((200, data)) - - -# TODO: Needs unit testing -class RoomMemberListRestServlet(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/members$") - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - # TODO support Pagination stream API (limit/tokens) - user = yield self.auth.get_user_by_req(request) - handler = self.handlers.room_member_handler - members = yield handler.get_room_members_as_pagination_chunk( - room_id=room_id, - user_id=user.to_string()) - - for event in members["chunk"]: - # FIXME: should probably be state_key here, not user_id - target_user = self.hs.parse_userid(event["user_id"]) - # Presence is an optional cache; don't fail if we can't fetch it - try: - presence_handler = self.handlers.presence_handler - presence_state = yield presence_handler.get_state( - target_user=target_user, auth_user=user - ) - event["content"].update(presence_state) - except: - pass - - defer.returnValue((200, members)) - - -# TODO: Needs unit testing -class RoomMessageListRestServlet(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/messages$") - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) - pagination_config = PaginationConfig.from_request( - request, default_limit=10, - ) - with_feedback = "feedback" in request.args - as_client_event = "raw" not in request.args - handler = self.handlers.message_handler - msgs = yield handler.get_messages( - room_id=room_id, - user_id=user.to_string(), - pagin_config=pagination_config, - feedback=with_feedback, - as_client_event=as_client_event - ) - - defer.returnValue((200, msgs)) - - -# TODO: Needs unit testing -class RoomStateRestServlet(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/state$") - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) - handler = self.handlers.message_handler - # Get all the current state for this room - events = yield handler.get_state_events( - room_id=room_id, - user_id=user.to_string(), - ) - defer.returnValue((200, events)) - - -# TODO: Needs unit testing -class RoomInitialSyncRestServlet(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/initialSync$") - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) - pagination_config = PaginationConfig.from_request(request) - content = yield self.handlers.message_handler.room_initial_sync( - room_id=room_id, - user_id=user.to_string(), - pagin_config=pagination_config, - ) - defer.returnValue((200, content)) - - -class RoomTriggerBackfill(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/backfill$") - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - remote_server = urllib.unquote( - request.args["remote"][0] - ).decode("UTF-8") - - limit = int(request.args["limit"][0]) - - handler = self.handlers.federation_handler - events = yield handler.backfill(remote_server, room_id, limit) - - res = [self.hs.serialize_event(event) for event in events] - defer.returnValue((200, res)) - - -# TODO: Needs unit testing -class RoomMembershipRestServlet(RestServlet): - - def register(self, http_server): - # /rooms/$roomid/[invite|join|leave] - PATTERN = ("/rooms/(?P[^/]*)/" - "(?Pjoin|invite|leave|ban|kick)") - register_txn_path(self, PATTERN, http_server) - - @defer.inlineCallbacks - def on_POST(self, request, room_id, membership_action): - user = yield self.auth.get_user_by_req(request) - - content = _parse_json(request) - - # target user is you unless it is an invite - state_key = user.to_string() - if membership_action in ["invite", "ban", "kick"]: - if "user_id" not in content: - raise SynapseError(400, "Missing user_id key.") - state_key = content["user_id"] - - if membership_action == "kick": - membership_action = "leave" - - msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event( - { - "type": EventTypes.Member, - "content": {"membership": unicode(membership_action)}, - "room_id": room_id, - "sender": user.to_string(), - "state_key": state_key, - } - ) - - defer.returnValue((200, {})) - - @defer.inlineCallbacks - def on_PUT(self, request, room_id, membership_action, txn_id): - try: - defer.returnValue( - self.txns.get_client_transaction(request, txn_id) - ) - except KeyError: - pass - - response = yield self.on_POST(request, room_id, membership_action) - - self.txns.store_client_transaction(request, txn_id, response) - defer.returnValue(response) - - -class RoomRedactEventRestServlet(RestServlet): - def register(self, http_server): - PATTERN = ("/rooms/(?P[^/]*)/redact/(?P[^/]*)") - register_txn_path(self, PATTERN, http_server) - - @defer.inlineCallbacks - def on_POST(self, request, room_id, event_id): - user = yield self.auth.get_user_by_req(request) - content = _parse_json(request) - - msg_handler = self.handlers.message_handler - event = yield msg_handler.create_and_send_event( - { - "type": EventTypes.Redaction, - "content": content, - "room_id": room_id, - "sender": user.to_string(), - "redacts": event_id, - } - ) - - defer.returnValue((200, {"event_id": event.event_id})) - - @defer.inlineCallbacks - def on_PUT(self, request, room_id, event_id, txn_id): - try: - defer.returnValue( - self.txns.get_client_transaction(request, txn_id) - ) - except KeyError: - pass - - response = yield self.on_POST(request, room_id, event_id) - - self.txns.store_client_transaction(request, txn_id, response) - defer.returnValue(response) - - -class RoomTypingRestServlet(RestServlet): - PATTERN = client_path_pattern( - "/rooms/(?P[^/]*)/typing/(?P[^/]*)$" - ) - - @defer.inlineCallbacks - def on_PUT(self, request, room_id, user_id): - auth_user = yield self.auth.get_user_by_req(request) - - room_id = urllib.unquote(room_id) - target_user = self.hs.parse_userid(urllib.unquote(user_id)) - - content = _parse_json(request) - - typing_handler = self.handlers.typing_notification_handler - - if content["typing"]: - yield typing_handler.started_typing( - target_user=target_user, - auth_user=auth_user, - room_id=room_id, - timeout=content.get("timeout", 30000), - ) - else: - yield typing_handler.stopped_typing( - target_user=target_user, - auth_user=auth_user, - room_id=room_id, - ) - - defer.returnValue((200, {})) - - -def _parse_json(request): - try: - content = json.loads(request.content.read()) - if type(content) != dict: - raise SynapseError(400, "Content must be a JSON object.", - errcode=Codes.NOT_JSON) - return content - except ValueError: - raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) - - -def register_txn_path(servlet, regex_string, http_server, with_get=False): - """Registers a transaction-based path. - - This registers two paths: - PUT regex_string/$txnid - POST regex_string - - Args: - regex_string (str): The regex string to register. Must NOT have a - trailing $ as this string will be appended to. - http_server : The http_server to register paths with. - with_get: True to also register respective GET paths for the PUTs. - """ - http_server.register_path( - "POST", - client_path_pattern(regex_string + "$"), - servlet.on_POST - ) - http_server.register_path( - "PUT", - client_path_pattern(regex_string + "/(?P[^/]*)$"), - servlet.on_PUT - ) - if with_get: - http_server.register_path( - "GET", - client_path_pattern(regex_string + "/(?P[^/]*)$"), - servlet.on_GET - ) - - -def register_servlets(hs, http_server): - RoomStateEventRestServlet(hs).register(http_server) - RoomCreateRestServlet(hs).register(http_server) - RoomMemberListRestServlet(hs).register(http_server) - RoomMessageListRestServlet(hs).register(http_server) - JoinRoomAliasServlet(hs).register(http_server) - RoomTriggerBackfill(hs).register(http_server) - RoomMembershipRestServlet(hs).register(http_server) - RoomSendEventRestServlet(hs).register(http_server) - PublicRoomListRestServlet(hs).register(http_server) - RoomStateRestServlet(hs).register(http_server) - RoomInitialSyncRestServlet(hs).register(http_server) - RoomRedactEventRestServlet(hs).register(http_server) - RoomTypingRestServlet(hs).register(http_server) diff --git a/synapse/client/v1/transactions.py b/synapse/client/v1/transactions.py deleted file mode 100644 index d933fea18a..0000000000 --- a/synapse/client/v1/transactions.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module contains logic for storing HTTP PUT transactions. This is used -to ensure idempotency when performing PUTs using the REST API.""" -import logging - -logger = logging.getLogger(__name__) - - -# FIXME: elsewhere we use FooStore to indicate something in the storage layer... -class HttpTransactionStore(object): - - def __init__(self): - # { key : (txn_id, response) } - self.transactions = {} - - def get_response(self, key, txn_id): - """Retrieve a response for this request. - - Args: - key (str): A transaction-independent key for this request. Usually - this is a combination of the path (without the transaction id) - and the user's access token. - txn_id (str): The transaction ID for this request - Returns: - A tuple of (HTTP response code, response content) or None. - """ - try: - logger.debug("get_response Key: %s TxnId: %s", key, txn_id) - (last_txn_id, response) = self.transactions[key] - if txn_id == last_txn_id: - logger.info("get_response: Returning a response for %s", key) - return response - except KeyError: - pass - return None - - def store_response(self, key, txn_id, response): - """Stores an HTTP response tuple. - - Args: - key (str): A transaction-independent key for this request. Usually - this is a combination of the path (without the transaction id) - and the user's access token. - txn_id (str): The transaction ID for this request. - response (tuple): A tuple of (HTTP response code, response content) - """ - logger.debug("store_response Key: %s TxnId: %s", key, txn_id) - self.transactions[key] = (txn_id, response) - - def store_client_transaction(self, request, txn_id, response): - """Stores the request/response pair of an HTTP transaction. - - Args: - request (twisted.web.http.Request): The twisted HTTP request. This - request must have the transaction ID as the last path segment. - response (tuple): A tuple of (response code, response dict) - txn_id (str): The transaction ID for this request. - """ - self.store_response(self._get_key(request), txn_id, response) - - def get_client_transaction(self, request, txn_id): - """Retrieves a stored response if there was one. - - Args: - request (twisted.web.http.Request): The twisted HTTP request. This - request must have the transaction ID as the last path segment. - txn_id (str): The transaction ID for this request. - Returns: - The response tuple. - Raises: - KeyError if the transaction was not found. - """ - response = self.get_response(self._get_key(request), txn_id) - if response is None: - raise KeyError("Transaction not found.") - return response - - def _get_key(self, request): - token = request.args["access_token"][0] - path_without_txn_id = request.path.rsplit("/", 1)[0] - return path_without_txn_id + "/" + token diff --git a/synapse/client/v1/voip.py b/synapse/client/v1/voip.py deleted file mode 100644 index 011c35e69b..0000000000 --- a/synapse/client/v1/voip.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from base import RestServlet, client_path_pattern - - -import hmac -import hashlib -import base64 - - -class VoipRestServlet(RestServlet): - PATTERN = client_path_pattern("/voip/turnServer$") - - @defer.inlineCallbacks - def on_GET(self, request): - auth_user = yield self.auth.get_user_by_req(request) - - turnUris = self.hs.config.turn_uris - turnSecret = self.hs.config.turn_shared_secret - userLifetime = self.hs.config.turn_user_lifetime - if not turnUris or not turnSecret or not userLifetime: - defer.returnValue((200, {})) - - expiry = (self.hs.get_clock().time_msec() + userLifetime) / 1000 - username = "%d:%s" % (expiry, auth_user.to_string()) - - mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1) - # We need to use standard base64 encoding here, *not* syutil's - # encode_base64 because we need to add the standard padding to get the - # same result as the TURN server. - password = base64.b64encode(mac.digest()) - - defer.returnValue((200, { - 'username': username, - 'password': password, - 'ttl': userLifetime / 1000, - 'uris': turnUris, - })) - - def on_OPTIONS(self, request): - return (200, {}) - - -def register_servlets(hs, http_server): - VoipRestServlet(hs).register(http_server) diff --git a/synapse/media/__init__.py b/synapse/media/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/synapse/media/v0/__init__.py b/synapse/media/v0/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/synapse/media/v0/content_repository.py b/synapse/media/v0/content_repository.py deleted file mode 100644 index 79ae0e3d74..0000000000 --- a/synapse/media/v0/content_repository.py +++ /dev/null @@ -1,212 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.http.server import respond_with_json_bytes - -from synapse.util.stringutils import random_string -from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException, Codes, cs_error -) - -from twisted.protocols.basic import FileSender -from twisted.web import server, resource -from twisted.internet import defer - -import base64 -import json -import logging -import os -import re - -logger = logging.getLogger(__name__) - - -class ContentRepoResource(resource.Resource): - """Provides file uploading and downloading. - - Uploads are POSTed to wherever this Resource is linked to. This resource - returns a "content token" which can be used to GET this content again. The - token is typically a path, but it may not be. Tokens can expire, be - one-time uses, etc. - - In this case, the token is a path to the file and contains 3 interesting - sections: - - User ID base64d (for namespacing content to each user) - - random 24 char string - - Content type base64d (so we can return it when clients GET it) - - """ - isLeaf = True - - def __init__(self, hs, directory, auth, external_addr): - resource.Resource.__init__(self) - self.hs = hs - self.directory = directory - self.auth = auth - self.external_addr = external_addr.rstrip('/') - self.max_upload_size = hs.config.max_upload_size - - if not os.path.isdir(self.directory): - os.mkdir(self.directory) - logger.info("ContentRepoResource : Created %s directory.", - self.directory) - - @defer.inlineCallbacks - def map_request_to_name(self, request): - # auth the user - auth_user = yield self.auth.get_user_by_req(request) - - # namespace all file uploads on the user - prefix = base64.urlsafe_b64encode( - auth_user.to_string() - ).replace('=', '') - - # use a random string for the main portion - main_part = random_string(24) - - # suffix with a file extension if we can make one. This is nice to - # provide a hint to clients on the file information. We will also reuse - # this info to spit back the content type to the client. - suffix = "" - if request.requestHeaders.hasHeader("Content-Type"): - content_type = request.requestHeaders.getRawHeaders( - "Content-Type")[0] - suffix = "." + base64.urlsafe_b64encode(content_type) - if (content_type.split("/")[0].lower() in - ["image", "video", "audio"]): - file_ext = content_type.split("/")[-1] - # be a little paranoid and only allow a-z - file_ext = re.sub("[^a-z]", "", file_ext) - suffix += "." + file_ext - - file_name = prefix + main_part + suffix - file_path = os.path.join(self.directory, file_name) - logger.info("User %s is uploading a file to path %s", - auth_user.to_string(), - file_path) - - # keep trying to make a non-clashing file, with a sensible max attempts - attempts = 0 - while os.path.exists(file_path): - main_part = random_string(24) - file_name = prefix + main_part + suffix - file_path = os.path.join(self.directory, file_name) - attempts += 1 - if attempts > 25: # really? Really? - raise SynapseError(500, "Unable to create file.") - - defer.returnValue(file_path) - - def render_GET(self, request): - # no auth here on purpose, to allow anyone to view, even across home - # servers. - - # TODO: A little crude here, we could do this better. - filename = request.path.split('/')[-1] - # be paranoid - filename = re.sub("[^0-9A-z.-_]", "", filename) - - file_path = self.directory + "/" + filename - - logger.debug("Searching for %s", file_path) - - if os.path.isfile(file_path): - # filename has the content type - base64_contentype = filename.split(".")[1] - content_type = base64.urlsafe_b64decode(base64_contentype) - logger.info("Sending file %s", file_path) - f = open(file_path, 'rb') - request.setHeader('Content-Type', content_type) - - # cache for at least a day. - # XXX: we might want to turn this off for data we don't want to - # recommend caching as it's sensitive or private - or at least - # select private. don't bother setting Expires as all our matrix - # clients are smart enough to be happy with Cache-Control (right?) - request.setHeader( - "Cache-Control", "public,max-age=86400,s-maxage=86400" - ) - - d = FileSender().beginFileTransfer(f, request) - - # after the file has been sent, clean up and finish the request - def cbFinished(ignored): - f.close() - request.finish() - d.addCallback(cbFinished) - else: - respond_with_json_bytes( - request, - 404, - json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)), - send_cors=True) - - return server.NOT_DONE_YET - - def render_POST(self, request): - self._async_render(request) - return server.NOT_DONE_YET - - def render_OPTIONS(self, request): - respond_with_json_bytes(request, 200, {}, send_cors=True) - return server.NOT_DONE_YET - - @defer.inlineCallbacks - def _async_render(self, request): - try: - # TODO: The checks here are a bit late. The content will have - # already been uploaded to a tmp file at this point - content_length = request.getHeader("Content-Length") - if content_length is None: - raise SynapseError( - msg="Request must specify a Content-Length", code=400 - ) - if int(content_length) > self.max_upload_size: - raise SynapseError( - msg="Upload request body is too large", - code=413, - ) - - fname = yield self.map_request_to_name(request) - - # TODO I have a suspicious feeling this is just going to block - with open(fname, "wb") as f: - f.write(request.content.read()) - - # FIXME (erikj): These should use constants. - file_name = os.path.basename(fname) - # FIXME: we can't assume what the repo's public mounted path is - # ...plus self-signed SSL won't work to remote clients anyway - # ...and we can't assume that it's SSL anyway, as we might want to - # serve it via the non-SSL listener... - url = "%s/_matrix/content/%s" % ( - self.external_addr, file_name - ) - - respond_with_json_bytes(request, 200, - json.dumps({"content_token": url}), - send_cors=True) - - except CodeMessageException as e: - logger.exception(e) - respond_with_json_bytes(request, e.code, - json.dumps(cs_exception(e))) - except Exception as e: - logger.error("Failed to store file: %s" % e) - respond_with_json_bytes( - request, - 500, - json.dumps({"error": "Internal server error"}), - send_cors=True) diff --git a/synapse/media/v1/__init__.py b/synapse/media/v1/__init__.py deleted file mode 100644 index d6c6690577..0000000000 --- a/synapse/media/v1/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import PIL.Image - -# check for JPEG support. -try: - PIL.Image._getdecoder("rgb", "jpeg", None) -except IOError as e: - if str(e).startswith("decoder jpeg not available"): - raise Exception( - "FATAL: jpeg codec not supported. Install pillow correctly! " - " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" - " pip install pillow --user'" - ) -except Exception: - # any other exception is fine - pass - - -# check for PNG support. -try: - PIL.Image._getdecoder("rgb", "zip", None) -except IOError as e: - if str(e).startswith("decoder zip not available"): - raise Exception( - "FATAL: zip codec not supported. Install pillow correctly! " - " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" - " pip install pillow --user'" - ) -except Exception: - # any other exception is fine - pass diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py deleted file mode 100644 index 688e7376ad..0000000000 --- a/synapse/media/v1/base_resource.py +++ /dev/null @@ -1,378 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .thumbnailer import Thumbnailer - -from synapse.http.server import respond_with_json -from synapse.util.stringutils import random_string -from synapse.api.errors import ( - cs_exception, CodeMessageException, cs_error, Codes, SynapseError -) - -from twisted.internet import defer -from twisted.web.resource import Resource -from twisted.protocols.basic import FileSender - -import os - -import logging - -logger = logging.getLogger(__name__) - - -class BaseMediaResource(Resource): - isLeaf = True - - def __init__(self, hs, filepaths): - Resource.__init__(self) - self.auth = hs.get_auth() - self.client = hs.get_http_client() - self.clock = hs.get_clock() - self.server_name = hs.hostname - self.store = hs.get_datastore() - self.max_upload_size = hs.config.max_upload_size - self.max_image_pixels = hs.config.max_image_pixels - self.filepaths = filepaths - self.downloads = {} - - @staticmethod - def catch_errors(request_handler): - @defer.inlineCallbacks - def wrapped_request_handler(self, request): - try: - yield request_handler(self, request) - except CodeMessageException as e: - logger.exception(e) - respond_with_json( - request, e.code, cs_exception(e), send_cors=True - ) - except: - logger.exception( - "Failed handle request %s.%s on %r", - request_handler.__module__, - request_handler.__name__, - self, - ) - respond_with_json( - request, - 500, - {"error": "Internal server error"}, - send_cors=True - ) - return wrapped_request_handler - - @staticmethod - def _parse_media_id(request): - try: - server_name, media_id = request.postpath - return (server_name, media_id) - except: - raise SynapseError( - 404, - "Invalid media id token %r" % (request.postpath,), - Codes.UNKKOWN, - ) - - @staticmethod - def _parse_integer(request, arg_name, default=None): - try: - if default is None: - return int(request.args[arg_name][0]) - else: - return int(request.args.get(arg_name, [default])[0]) - except: - raise SynapseError( - 400, - "Missing integer argument %r" % (arg_name,), - Codes.UNKNOWN, - ) - - @staticmethod - def _parse_string(request, arg_name, default=None): - try: - if default is None: - return request.args[arg_name][0] - else: - return request.args.get(arg_name, [default])[0] - except: - raise SynapseError( - 400, - "Missing string argument %r" % (arg_name,), - Codes.UNKNOWN, - ) - - def _respond_404(self, request): - respond_with_json( - request, 404, - cs_error( - "Not found %r" % (request.postpath,), - code=Codes.NOT_FOUND, - ), - send_cors=True - ) - - @staticmethod - def _makedirs(filepath): - dirname = os.path.dirname(filepath) - if not os.path.exists(dirname): - os.makedirs(dirname) - - def _get_remote_media(self, server_name, media_id): - key = (server_name, media_id) - download = self.downloads.get(key) - if download is None: - download = self._get_remote_media_impl(server_name, media_id) - self.downloads[key] = download - - @download.addBoth - def callback(media_info): - del self.downloads[key] - return media_info - return download - - @defer.inlineCallbacks - def _get_remote_media_impl(self, server_name, media_id): - media_info = yield self.store.get_cached_remote_media( - server_name, media_id - ) - if not media_info: - media_info = yield self._download_remote_file( - server_name, media_id - ) - defer.returnValue(media_info) - - @defer.inlineCallbacks - def _download_remote_file(self, server_name, media_id): - file_id = random_string(24) - - fname = self.filepaths.remote_media_filepath( - server_name, file_id - ) - self._makedirs(fname) - - try: - with open(fname, "wb") as f: - request_path = "/".join(( - "/_matrix/media/v1/download", server_name, media_id, - )) - length, headers = yield self.client.get_file( - server_name, request_path, output_stream=f, - max_size=self.max_upload_size, - ) - media_type = headers["Content-Type"][0] - time_now_ms = self.clock.time_msec() - - yield self.store.store_cached_remote_media( - origin=server_name, - media_id=media_id, - media_type=media_type, - time_now_ms=self.clock.time_msec(), - upload_name=None, - media_length=length, - filesystem_id=file_id, - ) - except: - os.remove(fname) - raise - - media_info = { - "media_type": media_type, - "media_length": length, - "upload_name": None, - "created_ts": time_now_ms, - "filesystem_id": file_id, - } - - yield self._generate_remote_thumbnails( - server_name, media_id, media_info - ) - - defer.returnValue(media_info) - - @defer.inlineCallbacks - def _respond_with_file(self, request, media_type, file_path, - file_size=None): - logger.debug("Responding with %r", file_path) - - if os.path.isfile(file_path): - request.setHeader(b"Content-Type", media_type.encode("UTF-8")) - - # cache for at least a day. - # XXX: we might want to turn this off for data we don't want to - # recommend caching as it's sensitive or private - or at least - # select private. don't bother setting Expires as all our - # clients are smart enough to be happy with Cache-Control - request.setHeader( - b"Cache-Control", b"public,max-age=86400,s-maxage=86400" - ) - if file_size is None: - stat = os.stat(file_path) - file_size = stat.st_size - - request.setHeader( - b"Content-Length", b"%d" % (file_size,) - ) - - with open(file_path, "rb") as f: - yield FileSender().beginFileTransfer(f, request) - - request.finish() - else: - self._respond_404(request) - - def _get_thumbnail_requirements(self, media_type): - if media_type == "image/jpeg": - return ( - (32, 32, "crop", "image/jpeg"), - (96, 96, "crop", "image/jpeg"), - (320, 240, "scale", "image/jpeg"), - (640, 480, "scale", "image/jpeg"), - ) - elif (media_type == "image/png") or (media_type == "image/gif"): - return ( - (32, 32, "crop", "image/png"), - (96, 96, "crop", "image/png"), - (320, 240, "scale", "image/png"), - (640, 480, "scale", "image/png"), - ) - else: - return () - - @defer.inlineCallbacks - def _generate_local_thumbnails(self, media_id, media_info): - media_type = media_info["media_type"] - requirements = self._get_thumbnail_requirements(media_type) - if not requirements: - return - - input_path = self.filepaths.local_media_filepath(media_id) - thumbnailer = Thumbnailer(input_path) - m_width = thumbnailer.width - m_height = thumbnailer.height - - if m_width * m_height >= self.max_image_pixels: - logger.info( - "Image too large to thumbnail %r x %r > %r", - m_width, m_height, self.max_image_pixels - ) - return - - scales = set() - crops = set() - for r_width, r_height, r_method, r_type in requirements: - if r_method == "scale": - t_width, t_height = thumbnailer.aspect(r_width, r_height) - scales.add(( - min(m_width, t_width), min(m_height, t_height), r_type, - )) - elif r_method == "crop": - crops.add((r_width, r_height, r_type)) - - for t_width, t_height, t_type in scales: - t_method = "scale" - t_path = self.filepaths.local_media_thumbnail( - media_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) - yield self.store.store_local_thumbnail( - media_id, t_width, t_height, t_type, t_method, t_len - ) - - for t_width, t_height, t_type in crops: - if (t_width, t_height, t_type) in scales: - # If the aspect ratio of the cropped thumbnail matches a purely - # scaled one then there is no point in calculating a separate - # thumbnail. - continue - t_method = "crop" - t_path = self.filepaths.local_media_thumbnail( - media_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) - yield self.store.store_local_thumbnail( - media_id, t_width, t_height, t_type, t_method, t_len - ) - - defer.returnValue({ - "width": m_width, - "height": m_height, - }) - - @defer.inlineCallbacks - def _generate_remote_thumbnails(self, server_name, media_id, media_info): - media_type = media_info["media_type"] - file_id = media_info["filesystem_id"] - requirements = self._get_thumbnail_requirements(media_type) - if not requirements: - return - - input_path = self.filepaths.remote_media_filepath(server_name, file_id) - thumbnailer = Thumbnailer(input_path) - m_width = thumbnailer.width - m_height = thumbnailer.height - - if m_width * m_height >= self.max_image_pixels: - logger.info( - "Image too large to thumbnail %r x %r > %r", - m_width, m_height, self.max_image_pixels - ) - return - - scales = set() - crops = set() - for r_width, r_height, r_method, r_type in requirements: - if r_method == "scale": - t_width, t_height = thumbnailer.aspect(r_width, r_height) - scales.add(( - min(m_width, t_width), min(m_height, t_height), r_type, - )) - elif r_method == "crop": - crops.add((r_width, r_height, r_type)) - - for t_width, t_height, t_type in scales: - t_method = "scale" - t_path = self.filepaths.remote_media_thumbnail( - server_name, file_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) - yield self.store.store_remote_media_thumbnail( - server_name, media_id, file_id, - t_width, t_height, t_type, t_method, t_len - ) - - for t_width, t_height, t_type in crops: - if (t_width, t_height, t_type) in scales: - # If the aspect ratio of the cropped thumbnail matches a purely - # scaled one then there is no point in calculating a separate - # thumbnail. - continue - t_method = "crop" - t_path = self.filepaths.remote_media_thumbnail( - server_name, file_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) - yield self.store.store_remote_media_thumbnail( - server_name, media_id, file_id, - t_width, t_height, t_type, t_method, t_len - ) - - defer.returnValue({ - "width": m_width, - "height": m_height, - }) diff --git a/synapse/media/v1/download_resource.py b/synapse/media/v1/download_resource.py deleted file mode 100644 index c585bb11f7..0000000000 --- a/synapse/media/v1/download_resource.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .base_resource import BaseMediaResource - -from twisted.web.server import NOT_DONE_YET -from twisted.internet import defer - -import logging - -logger = logging.getLogger(__name__) - - -class DownloadResource(BaseMediaResource): - def render_GET(self, request): - self._async_render_GET(request) - return NOT_DONE_YET - - @BaseMediaResource.catch_errors - @defer.inlineCallbacks - def _async_render_GET(self, request): - try: - server_name, media_id = request.postpath - except: - self._respond_404(request) - return - - if server_name == self.server_name: - yield self._respond_local_file(request, media_id) - else: - yield self._respond_remote_file(request, server_name, media_id) - - @defer.inlineCallbacks - def _respond_local_file(self, request, media_id): - media_info = yield self.store.get_local_media(media_id) - if not media_info: - self._respond_404(request) - return - - media_type = media_info["media_type"] - media_length = media_info["media_length"] - file_path = self.filepaths.local_media_filepath(media_id) - - yield self._respond_with_file( - request, media_type, file_path, media_length - ) - - @defer.inlineCallbacks - def _respond_remote_file(self, request, server_name, media_id): - media_info = yield self._get_remote_media(server_name, media_id) - - media_type = media_info["media_type"] - media_length = media_info["media_length"] - filesystem_id = media_info["filesystem_id"] - - file_path = self.filepaths.remote_media_filepath( - server_name, filesystem_id - ) - - yield self._respond_with_file( - request, media_type, file_path, media_length - ) diff --git a/synapse/media/v1/filepath.py b/synapse/media/v1/filepath.py deleted file mode 100644 index ed9a58e9d9..0000000000 --- a/synapse/media/v1/filepath.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - - -class MediaFilePaths(object): - - def __init__(self, base_path): - self.base_path = base_path - - def default_thumbnail(self, default_top_level, default_sub_type, width, - height, content_type, method): - top_level_type, sub_type = content_type.split("/") - file_name = "%i-%i-%s-%s-%s" % ( - width, height, top_level_type, sub_type, method - ) - return os.path.join( - self.base_path, "default_thumbnails", default_top_level, - default_sub_type, file_name - ) - - def local_media_filepath(self, media_id): - return os.path.join( - self.base_path, "local_content", - media_id[0:2], media_id[2:4], media_id[4:] - ) - - def local_media_thumbnail(self, media_id, width, height, content_type, - method): - top_level_type, sub_type = content_type.split("/") - file_name = "%i-%i-%s-%s-%s" % ( - width, height, top_level_type, sub_type, method - ) - return os.path.join( - self.base_path, "local_thumbnails", - media_id[0:2], media_id[2:4], media_id[4:], - file_name - ) - - def remote_media_filepath(self, server_name, file_id): - return os.path.join( - self.base_path, "remote_content", server_name, - file_id[0:2], file_id[2:4], file_id[4:] - ) - - def remote_media_thumbnail(self, server_name, file_id, width, height, - content_type, method): - top_level_type, sub_type = content_type.split("/") - file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type) - return os.path.join( - self.base_path, "remote_thumbnail", server_name, - file_id[0:2], file_id[2:4], file_id[4:], - file_name - ) diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py deleted file mode 100644 index 461cc001f1..0000000000 --- a/synapse/media/v1/media_repository.py +++ /dev/null @@ -1,77 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .upload_resource import UploadResource -from .download_resource import DownloadResource -from .thumbnail_resource import ThumbnailResource -from .filepath import MediaFilePaths - -from twisted.web.resource import Resource - -import logging - -logger = logging.getLogger(__name__) - - -class MediaRepositoryResource(Resource): - """File uploading and downloading. - - Uploads are POSTed to a resource which returns a token which is used to GET - the download:: - - => POST /_matrix/media/v1/upload HTTP/1.1 - Content-Type: - - - - <= HTTP/1.1 200 OK - Content-Type: application/json - - { "content_uri": "mxc:///" } - - => GET /_matrix/media/v1/download// HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: - Content-Disposition: attachment;filename= - - - - Clients can get thumbnails by supplying a desired width and height and - thumbnailing method:: - - => GET /_matrix/media/v1/thumbnail/ - /?width=&height=&method= HTTP/1.1 - - <= HTTP/1.1 200 OK - Content-Type: image/jpeg or image/png - - - - The thumbnail methods are "crop" and "scale". "scale" trys to return an - image where either the width or the height is smaller than the requested - size. The client should then scale and letterbox the image if it needs to - fit within a given rectangle. "crop" trys to return an image where the - width and height are close to the requested size and the aspect matches - the requested size. The client should scale the image if it needs to fit - within a given rectangle. - """ - - def __init__(self, hs): - Resource.__init__(self) - filepaths = MediaFilePaths(hs.config.media_store_path) - self.putChild("upload", UploadResource(hs, filepaths)) - self.putChild("download", DownloadResource(hs, filepaths)) - self.putChild("thumbnail", ThumbnailResource(hs, filepaths)) diff --git a/synapse/media/v1/thumbnail_resource.py b/synapse/media/v1/thumbnail_resource.py deleted file mode 100644 index 84f5e3463c..0000000000 --- a/synapse/media/v1/thumbnail_resource.py +++ /dev/null @@ -1,193 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from .base_resource import BaseMediaResource - -from twisted.web.server import NOT_DONE_YET -from twisted.internet import defer - -import logging - -logger = logging.getLogger(__name__) - - -class ThumbnailResource(BaseMediaResource): - isLeaf = True - - def render_GET(self, request): - self._async_render_GET(request) - return NOT_DONE_YET - - @BaseMediaResource.catch_errors - @defer.inlineCallbacks - def _async_render_GET(self, request): - server_name, media_id = self._parse_media_id(request) - width = self._parse_integer(request, "width") - height = self._parse_integer(request, "height") - method = self._parse_string(request, "method", "scale") - m_type = self._parse_string(request, "type", "image/png") - - if server_name == self.server_name: - yield self._respond_local_thumbnail( - request, media_id, width, height, method, m_type - ) - else: - yield self._respond_remote_thumbnail( - request, server_name, media_id, - width, height, method, m_type - ) - - @defer.inlineCallbacks - def _respond_local_thumbnail(self, request, media_id, width, height, - method, m_type): - media_info = yield self.store.get_local_media(media_id) - - if not media_info: - self._respond_404(request) - return - - thumbnail_infos = yield self.store.get_local_media_thumbnails(media_id) - - if thumbnail_infos: - thumbnail_info = self._select_thumbnail( - width, height, method, m_type, thumbnail_infos - ) - t_width = thumbnail_info["thumbnail_width"] - t_height = thumbnail_info["thumbnail_height"] - t_type = thumbnail_info["thumbnail_type"] - t_method = thumbnail_info["thumbnail_method"] - - file_path = self.filepaths.local_media_thumbnail( - media_id, t_width, t_height, t_type, t_method, - ) - yield self._respond_with_file(request, t_type, file_path) - - else: - yield self._respond_default_thumbnail( - request, media_info, width, height, method, m_type, - ) - - @defer.inlineCallbacks - def _respond_remote_thumbnail(self, request, server_name, media_id, width, - height, method, m_type): - # TODO: Don't download the whole remote file - # We should proxy the thumbnail from the remote server instead. - media_info = yield self._get_remote_media(server_name, media_id) - - thumbnail_infos = yield self.store.get_remote_media_thumbnails( - server_name, media_id, - ) - - if thumbnail_infos: - thumbnail_info = self._select_thumbnail( - width, height, method, m_type, thumbnail_infos - ) - t_width = thumbnail_info["thumbnail_width"] - t_height = thumbnail_info["thumbnail_height"] - t_type = thumbnail_info["thumbnail_type"] - t_method = thumbnail_info["thumbnail_method"] - file_id = thumbnail_info["filesystem_id"] - t_length = thumbnail_info["thumbnail_length"] - - file_path = self.filepaths.remote_media_thumbnail( - server_name, file_id, t_width, t_height, t_type, t_method, - ) - yield self._respond_with_file(request, t_type, file_path, t_length) - else: - yield self._respond_default_thumbnail( - request, media_info, width, height, method, m_type, - ) - - @defer.inlineCallbacks - def _respond_default_thumbnail(self, request, media_info, width, height, - method, m_type): - media_type = media_info["media_type"] - top_level_type = media_type.split("/")[0] - sub_type = media_type.split("/")[-1].split(";")[0] - thumbnail_infos = yield self.store.get_default_thumbnails( - top_level_type, sub_type, - ) - if not thumbnail_infos: - thumbnail_infos = yield self.store.get_default_thumbnails( - top_level_type, "_default", - ) - if not thumbnail_infos: - thumbnail_infos = yield self.store.get_default_thumbnails( - "_default", "_default", - ) - if not thumbnail_infos: - self._respond_404(request) - return - - thumbnail_info = self._select_thumbnail( - width, height, "crop", m_type, thumbnail_infos - ) - - t_width = thumbnail_info["thumbnail_width"] - t_height = thumbnail_info["thumbnail_height"] - t_type = thumbnail_info["thumbnail_type"] - t_method = thumbnail_info["thumbnail_method"] - t_length = thumbnail_info["thumbnail_length"] - - file_path = self.filepaths.default_thumbnail( - top_level_type, sub_type, t_width, t_height, t_type, t_method, - ) - yield self.respond_with_file(request, t_type, file_path, t_length) - - def _select_thumbnail(self, desired_width, desired_height, desired_method, - desired_type, thumbnail_infos): - d_w = desired_width - d_h = desired_height - - if desired_method.lower() == "crop": - info_list = [] - for info in thumbnail_infos: - t_w = info["thumbnail_width"] - t_h = info["thumbnail_height"] - t_method = info["thumbnail_method"] - if t_method == "scale" or t_method == "crop": - aspect_quality = abs(d_w * t_h - d_h * t_w) - size_quality = abs((d_w - t_w) * (d_h - t_h)) - type_quality = desired_type != info["thumbnail_type"] - length_quality = info["thumbnail_length"] - info_list.append(( - aspect_quality, size_quality, type_quality, - length_quality, info - )) - if info_list: - return min(info_list)[-1] - else: - info_list = [] - info_list2 = [] - for info in thumbnail_infos: - t_w = info["thumbnail_width"] - t_h = info["thumbnail_height"] - t_method = info["thumbnail_method"] - size_quality = abs((d_w - t_w) * (d_h - t_h)) - type_quality = desired_type != info["thumbnail_type"] - length_quality = info["thumbnail_length"] - if t_method == "scale" and (t_w >= d_w or t_h >= d_h): - info_list.append(( - size_quality, type_quality, length_quality, info - )) - elif t_method == "scale": - info_list2.append(( - size_quality, type_quality, length_quality, info - )) - if info_list: - return min(info_list)[-1] - else: - return min(info_list2)[-1] diff --git a/synapse/media/v1/thumbnailer.py b/synapse/media/v1/thumbnailer.py deleted file mode 100644 index 28404f2b7b..0000000000 --- a/synapse/media/v1/thumbnailer.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import PIL.Image as Image -from io import BytesIO - - -class Thumbnailer(object): - - FORMATS = { - "image/jpeg": "JPEG", - "image/png": "PNG", - } - - def __init__(self, input_path): - self.image = Image.open(input_path) - self.width, self.height = self.image.size - - def aspect(self, max_width, max_height): - """Calculate the largest size that preserves aspect ratio which - fits within the given rectangle:: - - (w_in / h_in) = (w_out / h_out) - w_out = min(w_max, h_max * (w_in / h_in)) - h_out = min(h_max, w_max * (h_in / w_in)) - - Args: - max_width: The largest possible width. - max_height: The larget possible height. - """ - - if max_width * self.height < max_height * self.width: - return (max_width, (max_width * self.height) // self.width) - else: - return ((max_height * self.width) // self.height, max_height) - - def scale(self, output_path, width, height, output_type): - """Rescales the image to the given dimensions""" - scaled = self.image.resize((width, height), Image.ANTIALIAS) - return self.save_image(scaled, output_type, output_path) - - def crop(self, output_path, width, height, output_type): - """Rescales and crops the image to the given dimensions preserving - aspect:: - (w_in / h_in) = (w_scaled / h_scaled) - w_scaled = max(w_out, h_out * (w_in / h_in)) - h_scaled = max(h_out, w_out * (h_in / w_in)) - - Args: - max_width: The largest possible width. - max_height: The larget possible height. - """ - if width * self.height > height * self.width: - scaled_height = (width * self.height) // self.width - scaled_image = self.image.resize( - (width, scaled_height), Image.ANTIALIAS - ) - crop_top = (scaled_height - height) // 2 - crop_bottom = height + crop_top - cropped = scaled_image.crop((0, crop_top, width, crop_bottom)) - else: - scaled_width = (height * self.width) // self.height - scaled_image = self.image.resize( - (scaled_width, height), Image.ANTIALIAS - ) - crop_left = (scaled_width - width) // 2 - crop_right = width + crop_left - cropped = scaled_image.crop((crop_left, 0, crop_right, height)) - return self.save_image(cropped, output_type, output_path) - - def save_image(self, output_image, output_type, output_path): - output_bytes_io = BytesIO() - output_image.save(output_bytes_io, self.FORMATS[output_type], quality=70) - output_bytes = output_bytes_io.getvalue() - with open(output_path, "wb") as output_file: - output_file.write(output_bytes) - return len(output_bytes) diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py deleted file mode 100644 index b1718a630b..0000000000 --- a/synapse/media/v1/upload_resource.py +++ /dev/null @@ -1,113 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.http.server import respond_with_json - -from synapse.util.stringutils import random_string -from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException -) - -from twisted.web.server import NOT_DONE_YET -from twisted.internet import defer - -from .base_resource import BaseMediaResource - -import logging - -logger = logging.getLogger(__name__) - - -class UploadResource(BaseMediaResource): - def render_POST(self, request): - self._async_render_POST(request) - return NOT_DONE_YET - - def render_OPTIONS(self, request): - respond_with_json(request, 200, {}, send_cors=True) - return NOT_DONE_YET - - @defer.inlineCallbacks - def _async_render_POST(self, request): - try: - auth_user = yield self.auth.get_user_by_req(request) - # TODO: The checks here are a bit late. The content will have - # already been uploaded to a tmp file at this point - content_length = request.getHeader("Content-Length") - if content_length is None: - raise SynapseError( - msg="Request must specify a Content-Length", code=400 - ) - if int(content_length) > self.max_upload_size: - raise SynapseError( - msg="Upload request body is too large", - code=413, - ) - - headers = request.requestHeaders - - if headers.hasHeader("Content-Type"): - media_type = headers.getRawHeaders("Content-Type")[0] - else: - raise SynapseError( - msg="Upload request missing 'Content-Type'", - code=400, - ) - - #if headers.hasHeader("Content-Disposition"): - # disposition = headers.getRawHeaders("Content-Disposition")[0] - # TODO(markjh): parse content-dispostion - - media_id = random_string(24) - - fname = self.filepaths.local_media_filepath(media_id) - self._makedirs(fname) - - # This shouldn't block for very long because the content will have - # already been uploaded at this point. - with open(fname, "wb") as f: - f.write(request.content.read()) - - yield self.store.store_local_media( - media_id=media_id, - media_type=media_type, - time_now_ms=self.clock.time_msec(), - upload_name=None, - media_length=content_length, - user_id=auth_user, - ) - media_info = { - "media_type": media_type, - "media_length": content_length, - } - - yield self._generate_local_thumbnails(media_id, media_info) - - content_uri = "mxc://%s/%s" % (self.server_name, media_id) - - respond_with_json( - request, 200, {"content_uri": content_uri}, send_cors=True - ) - except CodeMessageException as e: - logger.exception(e) - respond_with_json(request, e.code, cs_exception(e), send_cors=True) - except: - logger.exception("Failed to store file") - respond_with_json( - request, - 500, - {"error": "Internal server error"}, - send_cors=True - ) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/synapse/rest/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/rest/client/__init__.py b/synapse/rest/client/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/synapse/rest/client/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py new file mode 100644 index 0000000000..88ec9cd27d --- /dev/null +++ b/synapse/rest/client/v1/__init__.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from . import ( + room, events, register, login, profile, presence, initial_sync, directory, + voip, admin, +) + + +class RestServletFactory(object): + + """ A factory for creating REST servlets. + + These REST servlets represent the entire client-server REST API. Generally + speaking, they serve as wrappers around events and the handlers that + process them. + + See synapse.events for information on synapse events. + """ + + def __init__(self, hs): + client_resource = hs.get_resource_for_client() + + # TODO(erikj): There *must* be a better way of doing this. + room.register_servlets(hs, client_resource) + events.register_servlets(hs, client_resource) + register.register_servlets(hs, client_resource) + login.register_servlets(hs, client_resource) + profile.register_servlets(hs, client_resource) + presence.register_servlets(hs, client_resource) + initial_sync.register_servlets(hs, client_resource) + directory.register_servlets(hs, client_resource) + voip.register_servlets(hs, client_resource) + admin.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py new file mode 100644 index 0000000000..0aa83514c8 --- /dev/null +++ b/synapse/rest/client/v1/admin.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import AuthError, SynapseError +from base import RestServlet, client_path_pattern + +import logging + +logger = logging.getLogger(__name__) + + +class WhoisRestServlet(RestServlet): + PATTERN = client_path_pattern("/admin/whois/(?P[^/]*)") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + target_user = self.hs.parse_userid(user_id) + auth_user = yield self.auth.get_user_by_req(request) + is_admin = yield self.auth.is_server_admin(auth_user) + + if not is_admin and target_user != auth_user: + raise AuthError(403, "You are not a server admin") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only whois a local user") + + ret = yield self.handlers.admin_handler.get_whois(target_user) + + defer.returnValue((200, ret)) + + +def register_servlets(hs, http_server): + WhoisRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1/base.py b/synapse/rest/client/v1/base.py new file mode 100644 index 0000000000..d005206b77 --- /dev/null +++ b/synapse/rest/client/v1/base.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains base REST classes for constructing REST servlets. """ +from synapse.api.urls import CLIENT_PREFIX +from .transactions import HttpTransactionStore +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def client_path_pattern(path_regex): + """Creates a regex compiled client path with the correct client path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + CLIENT_PREFIX + path_regex) + + +class RestServlet(object): + + """ A Synapse REST Servlet. + + An implementing class can either provide its own custom 'register' method, + or use the automatic pattern handling provided by the base class. + + To use this latter, the implementing class instead provides a `PATTERN` + class attribute containing a pre-compiled regular expression. The automatic + register method will then use this method to register any of the following + instance methods associated with the corresponding HTTP method: + + on_GET + on_PUT + on_POST + on_DELETE + on_OPTIONS + + Automatically handles turning CodeMessageExceptions thrown by these methods + into the appropriate HTTP response. + """ + + def __init__(self, hs): + self.hs = hs + + self.handlers = hs.get_handlers() + self.builder_factory = hs.get_event_builder_factory() + self.auth = hs.get_auth() + self.txns = HttpTransactionStore() + + def register(self, http_server): + """ Register this servlet with the given HTTP server. """ + if hasattr(self, "PATTERN"): + pattern = self.PATTERN + + for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"): + if hasattr(self, "on_%s" % (method)): + method_handler = getattr(self, "on_%s" % (method)) + http_server.register_path(method, pattern, method_handler) + else: + raise NotImplementedError("RestServlet must register something.") diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py new file mode 100644 index 0000000000..7ff44fdd9e --- /dev/null +++ b/synapse/rest/client/v1/directory.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from synapse.api.errors import AuthError, SynapseError, Codes +from base import RestServlet, client_path_pattern + +import json +import logging + + +logger = logging.getLogger(__name__) + + +def register_servlets(hs, http_server): + ClientDirectoryServer(hs).register(http_server) + + +class ClientDirectoryServer(RestServlet): + PATTERN = client_path_pattern("/directory/room/(?P[^/]*)$") + + @defer.inlineCallbacks + def on_GET(self, request, room_alias): + room_alias = self.hs.parse_roomalias(room_alias) + + dir_handler = self.handlers.directory_handler + res = yield dir_handler.get_association(room_alias) + + defer.returnValue((200, res)) + + @defer.inlineCallbacks + def on_PUT(self, request, room_alias): + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + if not "room_id" in content: + raise SynapseError(400, "Missing room_id key", + errcode=Codes.BAD_JSON) + + logger.debug("Got content: %s", content) + + room_alias = self.hs.parse_roomalias(room_alias) + + logger.debug("Got room name: %s", room_alias.to_string()) + + room_id = content["room_id"] + servers = content["servers"] if "servers" in content else None + + logger.debug("Got room_id: %s", room_id) + logger.debug("Got servers: %s", servers) + + # TODO(erikj): Check types. + # TODO(erikj): Check that room exists + + dir_handler = self.handlers.directory_handler + + try: + user_id = user.to_string() + yield dir_handler.create_association( + user_id, room_alias, room_id, servers + ) + yield dir_handler.send_room_alias_update_event(user_id, room_id) + except SynapseError as e: + raise e + except: + logger.exception("Failed to create association") + raise + + defer.returnValue((200, {})) + + @defer.inlineCallbacks + def on_DELETE(self, request, room_alias): + user = yield self.auth.get_user_by_req(request) + + is_admin = yield self.auth.is_server_admin(user) + if not is_admin: + raise AuthError(403, "You need to be a server admin") + + dir_handler = self.handlers.directory_handler + + room_alias = self.hs.parse_roomalias(room_alias) + + yield dir_handler.delete_association( + user.to_string(), room_alias + ) + + defer.returnValue((200, {})) + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py new file mode 100644 index 0000000000..c2515528ac --- /dev/null +++ b/synapse/rest/client/v1/events.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains REST servlets to do with event streaming, /events.""" +from twisted.internet import defer + +from synapse.api.errors import SynapseError +from synapse.streams.config import PaginationConfig +from .base import RestServlet, client_path_pattern + +import logging + + +logger = logging.getLogger(__name__) + + +class EventStreamRestServlet(RestServlet): + PATTERN = client_path_pattern("/events$") + + DEFAULT_LONGPOLL_TIME_MS = 30000 + + @defer.inlineCallbacks + def on_GET(self, request): + auth_user = yield self.auth.get_user_by_req(request) + try: + handler = self.handlers.event_stream_handler + pagin_config = PaginationConfig.from_request(request) + timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS + if "timeout" in request.args: + try: + timeout = int(request.args["timeout"][0]) + except ValueError: + raise SynapseError(400, "timeout must be in milliseconds.") + + as_client_event = "raw" not in request.args + + chunk = yield handler.get_stream( + auth_user.to_string(), pagin_config, timeout=timeout, + as_client_event=as_client_event + ) + except: + logger.exception("Event stream failed") + raise + + defer.returnValue((200, chunk)) + + def on_OPTIONS(self, request): + return (200, {}) + + +# TODO: Unit test gets, with and without auth, with different kinds of events. +class EventRestServlet(RestServlet): + PATTERN = client_path_pattern("/events/(?P[^/]*)$") + + @defer.inlineCallbacks + def on_GET(self, request, event_id): + auth_user = yield self.auth.get_user_by_req(request) + handler = self.handlers.event_handler + event = yield handler.get_event(auth_user, event_id) + + if event: + defer.returnValue((200, self.hs.serialize_event(event))) + else: + defer.returnValue((404, "Event not found.")) + + +def register_servlets(hs, http_server): + EventStreamRestServlet(hs).register(http_server) + EventRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py new file mode 100644 index 0000000000..b13d56b286 --- /dev/null +++ b/synapse/rest/client/v1/initial_sync.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.streams.config import PaginationConfig +from base import RestServlet, client_path_pattern + + +# TODO: Needs unit testing +class InitialSyncRestServlet(RestServlet): + PATTERN = client_path_pattern("/initialSync$") + + @defer.inlineCallbacks + def on_GET(self, request): + user = yield self.auth.get_user_by_req(request) + with_feedback = "feedback" in request.args + as_client_event = "raw" not in request.args + pagination_config = PaginationConfig.from_request(request) + handler = self.handlers.message_handler + content = yield handler.snapshot_all_rooms( + user_id=user.to_string(), + pagin_config=pagination_config, + feedback=with_feedback, + as_client_event=as_client_event + ) + + defer.returnValue((200, content)) + + +def register_servlets(hs, http_server): + InitialSyncRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py new file mode 100644 index 0000000000..6b8deff67b --- /dev/null +++ b/synapse/rest/client/v1/login.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import SynapseError +from synapse.types import UserID +from base import RestServlet, client_path_pattern + +import json + + +class LoginRestServlet(RestServlet): + PATTERN = client_path_pattern("/login$") + PASS_TYPE = "m.login.password" + + def on_GET(self, request): + return (200, {"flows": [{"type": LoginRestServlet.PASS_TYPE}]}) + + def on_OPTIONS(self, request): + return (200, {}) + + @defer.inlineCallbacks + def on_POST(self, request): + login_submission = _parse_json(request) + try: + if login_submission["type"] == LoginRestServlet.PASS_TYPE: + result = yield self.do_password_login(login_submission) + defer.returnValue(result) + else: + raise SynapseError(400, "Bad login type.") + except KeyError: + raise SynapseError(400, "Missing JSON keys.") + + @defer.inlineCallbacks + def do_password_login(self, login_submission): + if not login_submission["user"].startswith('@'): + login_submission["user"] = UserID.create( + login_submission["user"], self.hs.hostname).to_string() + + handler = self.handlers.login_handler + token = yield handler.login( + user=login_submission["user"], + password=login_submission["password"]) + + result = { + "user_id": login_submission["user"], # may have changed + "access_token": token, + "home_server": self.hs.hostname, + } + + defer.returnValue((200, result)) + + +class LoginFallbackRestServlet(RestServlet): + PATTERN = client_path_pattern("/login/fallback$") + + def on_GET(self, request): + # TODO(kegan): This should be returning some HTML which is capable of + # hitting LoginRestServlet + return (200, {}) + + +class PasswordResetRestServlet(RestServlet): + PATTERN = client_path_pattern("/login/reset") + + @defer.inlineCallbacks + def on_POST(self, request): + reset_info = _parse_json(request) + try: + email = reset_info["email"] + user_id = reset_info["user_id"] + handler = self.handlers.login_handler + yield handler.reset_password(user_id, email) + # purposefully give no feedback to avoid people hammering different + # combinations. + defer.returnValue((200, {})) + except KeyError: + raise SynapseError( + 400, + "Missing keys. Requires 'email' and 'user_id'." + ) + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.") + return content + except ValueError: + raise SynapseError(400, "Content not JSON.") + + +def register_servlets(hs, http_server): + LoginRestServlet(hs).register(http_server) + # TODO PasswordResetRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py new file mode 100644 index 0000000000..ca4d2d21f0 --- /dev/null +++ b/synapse/rest/client/v1/presence.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains REST servlets to do with presence: /presence/ +""" +from twisted.internet import defer + +from synapse.api.errors import SynapseError +from base import RestServlet, client_path_pattern + +import json +import logging + +logger = logging.getLogger(__name__) + + +class PresenceStatusRestServlet(RestServlet): + PATTERN = client_path_pattern("/presence/(?P[^/]*)/status") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + state = yield self.handlers.presence_handler.get_state( + target_user=user, auth_user=auth_user) + + defer.returnValue((200, state)) + + @defer.inlineCallbacks + def on_PUT(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + state = {} + try: + content = json.loads(request.content.read()) + + state["presence"] = content.pop("presence") + + if "status_msg" in content: + state["status_msg"] = content.pop("status_msg") + if not isinstance(state["status_msg"], basestring): + raise SynapseError(400, "status_msg must be a string.") + + if content: + raise KeyError() + except SynapseError as e: + raise e + except: + raise SynapseError(400, "Unable to parse state") + + yield self.handlers.presence_handler.set_state( + target_user=user, auth_user=auth_user, state=state) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request): + return (200, {}) + + +class PresenceListRestServlet(RestServlet): + PATTERN = client_path_pattern("/presence/list/(?P[^/]*)") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + if not self.hs.is_mine(user): + raise SynapseError(400, "User not hosted on this Home Server") + + if auth_user != user: + raise SynapseError(400, "Cannot get another user's presence list") + + presence = yield self.handlers.presence_handler.get_presence_list( + observer_user=user, accepted=True) + + for p in presence: + observed_user = p.pop("observed_user") + p["user_id"] = observed_user.to_string() + + defer.returnValue((200, presence)) + + @defer.inlineCallbacks + def on_POST(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + if not self.hs.is_mine(user): + raise SynapseError(400, "User not hosted on this Home Server") + + if auth_user != user: + raise SynapseError( + 400, "Cannot modify another user's presence list") + + try: + content = json.loads(request.content.read()) + except: + logger.exception("JSON parse error") + raise SynapseError(400, "Unable to parse content") + + if "invite" in content: + for u in content["invite"]: + if not isinstance(u, basestring): + raise SynapseError(400, "Bad invite value.") + if len(u) == 0: + continue + invited_user = self.hs.parse_userid(u) + yield self.handlers.presence_handler.send_invite( + observer_user=user, observed_user=invited_user + ) + + if "drop" in content: + for u in content["drop"]: + if not isinstance(u, basestring): + raise SynapseError(400, "Bad drop value.") + if len(u) == 0: + continue + dropped_user = self.hs.parse_userid(u) + yield self.handlers.presence_handler.drop( + observer_user=user, observed_user=dropped_user + ) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request): + return (200, {}) + + +def register_servlets(hs, http_server): + PresenceStatusRestServlet(hs).register(http_server) + PresenceListRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py new file mode 100644 index 0000000000..dc6eb424b0 --- /dev/null +++ b/synapse/rest/client/v1/profile.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains REST servlets to do with profile: /profile/ """ +from twisted.internet import defer + +from base import RestServlet, client_path_pattern + +import json + + +class ProfileDisplaynameRestServlet(RestServlet): + PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + user = self.hs.parse_userid(user_id) + + displayname = yield self.handlers.profile_handler.get_displayname( + user, + ) + + defer.returnValue((200, {"displayname": displayname})) + + @defer.inlineCallbacks + def on_PUT(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + try: + content = json.loads(request.content.read()) + new_name = content["displayname"] + except: + defer.returnValue((400, "Unable to parse name")) + + yield self.handlers.profile_handler.set_displayname( + user, auth_user, new_name) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request, user_id): + return (200, {}) + + +class ProfileAvatarURLRestServlet(RestServlet): + PATTERN = client_path_pattern("/profile/(?P[^/]*)/avatar_url") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + user = self.hs.parse_userid(user_id) + + avatar_url = yield self.handlers.profile_handler.get_avatar_url( + user, + ) + + defer.returnValue((200, {"avatar_url": avatar_url})) + + @defer.inlineCallbacks + def on_PUT(self, request, user_id): + auth_user = yield self.auth.get_user_by_req(request) + user = self.hs.parse_userid(user_id) + + try: + content = json.loads(request.content.read()) + new_name = content["avatar_url"] + except: + defer.returnValue((400, "Unable to parse name")) + + yield self.handlers.profile_handler.set_avatar_url( + user, auth_user, new_name) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request, user_id): + return (200, {}) + + +class ProfileRestServlet(RestServlet): + PATTERN = client_path_pattern("/profile/(?P[^/]*)") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + user = self.hs.parse_userid(user_id) + + displayname = yield self.handlers.profile_handler.get_displayname( + user, + ) + avatar_url = yield self.handlers.profile_handler.get_avatar_url( + user, + ) + + defer.returnValue((200, { + "displayname": displayname, + "avatar_url": avatar_url + })) + + +def register_servlets(hs, http_server): + ProfileDisplaynameRestServlet(hs).register(http_server) + ProfileAvatarURLRestServlet(hs).register(http_server) + ProfileRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py new file mode 100644 index 0000000000..e3b26902d9 --- /dev/null +++ b/synapse/rest/client/v1/register.py @@ -0,0 +1,291 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains REST servlets to do with registration: /register""" +from twisted.internet import defer + +from synapse.api.errors import SynapseError, Codes +from synapse.api.constants import LoginType +from base import RestServlet, client_path_pattern +import synapse.util.stringutils as stringutils + +from synapse.util.async import run_on_reactor + +from hashlib import sha1 +import hmac +import json +import logging +import urllib + +logger = logging.getLogger(__name__) + + +# We ought to be using hmac.compare_digest() but on older pythons it doesn't +# exist. It's a _really minor_ security flaw to use plain string comparison +# because the timing attack is so obscured by all the other code here it's +# unlikely to make much difference +if hasattr(hmac, "compare_digest"): + compare_digest = hmac.compare_digest +else: + compare_digest = lambda a, b: a == b + + +class RegisterRestServlet(RestServlet): + """Handles registration with the home server. + + This servlet is in control of the registration flow; the registration + handler doesn't have a concept of multi-stages or sessions. + """ + + PATTERN = client_path_pattern("/register$") + + def __init__(self, hs): + super(RegisterRestServlet, self).__init__(hs) + # sessions are stored as: + # self.sessions = { + # "session_id" : { __session_dict__ } + # } + # TODO: persistent storage + self.sessions = {} + + def on_GET(self, request): + if self.hs.config.enable_registration_captcha: + return ( + 200, + {"flows": [ + { + "type": LoginType.RECAPTCHA, + "stages": [ + LoginType.RECAPTCHA, + LoginType.EMAIL_IDENTITY, + LoginType.PASSWORD + ] + }, + { + "type": LoginType.RECAPTCHA, + "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD] + } + ]} + ) + else: + return ( + 200, + {"flows": [ + { + "type": LoginType.EMAIL_IDENTITY, + "stages": [ + LoginType.EMAIL_IDENTITY, LoginType.PASSWORD + ] + }, + { + "type": LoginType.PASSWORD + } + ]} + ) + + @defer.inlineCallbacks + def on_POST(self, request): + register_json = _parse_json(request) + + session = (register_json["session"] + if "session" in register_json else None) + login_type = None + if "type" not in register_json: + raise SynapseError(400, "Missing 'type' key.") + + try: + login_type = register_json["type"] + stages = { + LoginType.RECAPTCHA: self._do_recaptcha, + LoginType.PASSWORD: self._do_password, + LoginType.EMAIL_IDENTITY: self._do_email_identity + } + + session_info = self._get_session_info(request, session) + logger.debug("%s : session info %s request info %s", + login_type, session_info, register_json) + response = yield stages[login_type]( + request, + register_json, + session_info + ) + + if "access_token" not in response: + # isn't a final response + response["session"] = session_info["id"] + + defer.returnValue((200, response)) + except KeyError as e: + logger.exception(e) + raise SynapseError(400, "Missing JSON keys for login type %s." % ( + login_type, + )) + + def on_OPTIONS(self, request): + return (200, {}) + + def _get_session_info(self, request, session_id): + if not session_id: + # create a new session + while session_id is None or session_id in self.sessions: + session_id = stringutils.random_string(24) + self.sessions[session_id] = { + "id": session_id, + LoginType.EMAIL_IDENTITY: False, + LoginType.RECAPTCHA: False + } + + return self.sessions[session_id] + + def _save_session(self, session): + # TODO: Persistent storage + logger.debug("Saving session %s", session) + self.sessions[session["id"]] = session + + def _remove_session(self, session): + logger.debug("Removing session %s", session) + self.sessions.pop(session["id"]) + + @defer.inlineCallbacks + def _do_recaptcha(self, request, register_json, session): + if not self.hs.config.enable_registration_captcha: + raise SynapseError(400, "Captcha not required.") + + yield self._check_recaptcha(request, register_json, session) + + session[LoginType.RECAPTCHA] = True # mark captcha as done + self._save_session(session) + defer.returnValue({ + "next": [LoginType.PASSWORD, LoginType.EMAIL_IDENTITY] + }) + + @defer.inlineCallbacks + def _check_recaptcha(self, request, register_json, session): + if ("captcha_bypass_hmac" in register_json and + self.hs.config.captcha_bypass_secret): + if "user" not in register_json: + raise SynapseError(400, "Captcha bypass needs 'user'") + + want = hmac.new( + key=self.hs.config.captcha_bypass_secret, + msg=register_json["user"], + digestmod=sha1, + ).hexdigest() + + # str() because otherwise hmac complains that 'unicode' does not + # have the buffer interface + got = str(register_json["captcha_bypass_hmac"]) + + if compare_digest(want, got): + session["user"] = register_json["user"] + defer.returnValue(None) + else: + raise SynapseError( + 400, "Captcha bypass HMAC incorrect", + errcode=Codes.CAPTCHA_NEEDED + ) + + challenge = None + user_response = None + try: + challenge = register_json["challenge"] + user_response = register_json["response"] + except KeyError: + raise SynapseError(400, "Captcha response is required", + errcode=Codes.CAPTCHA_NEEDED) + + ip_addr = self.hs.get_ip_from_request(request) + + handler = self.handlers.registration_handler + yield handler.check_recaptcha( + ip_addr, + self.hs.config.recaptcha_private_key, + challenge, + user_response + ) + + @defer.inlineCallbacks + def _do_email_identity(self, request, register_json, session): + if (self.hs.config.enable_registration_captcha and + not session[LoginType.RECAPTCHA]): + raise SynapseError(400, "Captcha is required.") + + threepidCreds = register_json['threepidCreds'] + handler = self.handlers.registration_handler + logger.debug("Registering email. threepidcreds: %s" % (threepidCreds)) + yield handler.register_email(threepidCreds) + session["threepidCreds"] = threepidCreds # store creds for next stage + session[LoginType.EMAIL_IDENTITY] = True # mark email as done + self._save_session(session) + defer.returnValue({ + "next": LoginType.PASSWORD + }) + + @defer.inlineCallbacks + def _do_password(self, request, register_json, session): + yield run_on_reactor() + if (self.hs.config.enable_registration_captcha and + not session[LoginType.RECAPTCHA]): + # captcha should've been done by this stage! + raise SynapseError(400, "Captcha is required.") + + if ("user" in session and "user" in register_json and + session["user"] != register_json["user"]): + raise SynapseError( + 400, "Cannot change user ID during registration" + ) + + password = register_json["password"].encode("utf-8") + desired_user_id = (register_json["user"].encode("utf-8") + if "user" in register_json else None) + if (desired_user_id + and urllib.quote(desired_user_id) != desired_user_id): + raise SynapseError( + 400, + "User ID must only contain characters which do not " + + "require URL encoding.") + handler = self.handlers.registration_handler + (user_id, token) = yield handler.register( + localpart=desired_user_id, + password=password + ) + + if session[LoginType.EMAIL_IDENTITY]: + logger.debug("Binding emails %s to %s" % ( + session["threepidCreds"], user_id) + ) + yield handler.bind_emails(user_id, session["threepidCreds"]) + + result = { + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + } + self._remove_session(session) + defer.returnValue(result) + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.") + return content + except ValueError: + raise SynapseError(400, "Content not JSON.") + + +def register_servlets(hs, http_server): + RegisterRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py new file mode 100644 index 0000000000..48bba2a5f3 --- /dev/null +++ b/synapse/rest/client/v1/room.py @@ -0,0 +1,559 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains REST servlets to do with rooms: /rooms/ """ +from twisted.internet import defer + +from base import RestServlet, client_path_pattern +from synapse.api.errors import SynapseError, Codes +from synapse.streams.config import PaginationConfig +from synapse.api.constants import EventTypes, Membership + +import json +import logging +import urllib + + +logger = logging.getLogger(__name__) + + +class RoomCreateRestServlet(RestServlet): + # No PATTERN; we have custom dispatch rules here + + def register(self, http_server): + PATTERN = "/createRoom" + register_txn_path(self, PATTERN, http_server) + # define CORS for all of /rooms in RoomCreateRestServlet for simplicity + http_server.register_path("OPTIONS", + client_path_pattern("/rooms(?:/.*)?$"), + self.on_OPTIONS) + # define CORS for /createRoom[/txnid] + http_server.register_path("OPTIONS", + client_path_pattern("/createRoom(?:/.*)?$"), + self.on_OPTIONS) + + @defer.inlineCallbacks + def on_PUT(self, request, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + + response = yield self.on_POST(request) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + + @defer.inlineCallbacks + def on_POST(self, request): + auth_user = yield self.auth.get_user_by_req(request) + + room_config = self.get_room_config(request) + info = yield self.make_room(room_config, auth_user, None) + room_config.update(info) + defer.returnValue((200, info)) + + @defer.inlineCallbacks + def make_room(self, room_config, auth_user, room_id): + handler = self.handlers.room_creation_handler + info = yield handler.create_room( + user_id=auth_user.to_string(), + room_id=room_id, + config=room_config + ) + defer.returnValue(info) + + def get_room_config(self, request): + try: + user_supplied_config = json.loads(request.content.read()) + if "visibility" not in user_supplied_config: + # default visibility + user_supplied_config["visibility"] = "public" + return user_supplied_config + except (ValueError, TypeError): + raise SynapseError(400, "Body must be JSON.", + errcode=Codes.BAD_JSON) + + def on_OPTIONS(self, request): + return (200, {}) + + +# TODO: Needs unit testing for generic events +class RoomStateEventRestServlet(RestServlet): + def register(self, http_server): + # /room/$roomid/state/$eventtype + no_state_key = "/rooms/(?P[^/]*)/state/(?P[^/]*)$" + + # /room/$roomid/state/$eventtype/$statekey + state_key = ("/rooms/(?P[^/]*)/state/" + "(?P[^/]*)/(?P[^/]*)$") + + http_server.register_path("GET", + client_path_pattern(state_key), + self.on_GET) + http_server.register_path("PUT", + client_path_pattern(state_key), + self.on_PUT) + http_server.register_path("GET", + client_path_pattern(no_state_key), + self.on_GET_no_state_key) + http_server.register_path("PUT", + client_path_pattern(no_state_key), + self.on_PUT_no_state_key) + + def on_GET_no_state_key(self, request, room_id, event_type): + return self.on_GET(request, room_id, event_type, "") + + def on_PUT_no_state_key(self, request, room_id, event_type): + return self.on_PUT(request, room_id, event_type, "") + + @defer.inlineCallbacks + def on_GET(self, request, room_id, event_type, state_key): + user = yield self.auth.get_user_by_req(request) + + msg_handler = self.handlers.message_handler + data = yield msg_handler.get_room_data( + user_id=user.to_string(), + room_id=room_id, + event_type=event_type, + state_key=state_key, + ) + + if not data: + raise SynapseError( + 404, "Event not found.", errcode=Codes.NOT_FOUND + ) + defer.returnValue((200, data.get_dict()["content"])) + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, event_type, state_key): + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + event_dict = { + "type": event_type, + "content": content, + "room_id": room_id, + "sender": user.to_string(), + } + + if state_key is not None: + event_dict["state_key"] = state_key + + msg_handler = self.handlers.message_handler + yield msg_handler.create_and_send_event(event_dict) + + defer.returnValue((200, {})) + + +# TODO: Needs unit testing for generic events + feedback +class RoomSendEventRestServlet(RestServlet): + + def register(self, http_server): + # /rooms/$roomid/send/$event_type[/$txn_id] + PATTERN = ("/rooms/(?P[^/]*)/send/(?P[^/]*)") + register_txn_path(self, PATTERN, http_server, with_get=True) + + @defer.inlineCallbacks + def on_POST(self, request, room_id, event_type): + user = yield self.auth.get_user_by_req(request) + content = _parse_json(request) + + msg_handler = self.handlers.message_handler + event = yield msg_handler.create_and_send_event( + { + "type": event_type, + "content": content, + "room_id": room_id, + "sender": user.to_string(), + } + ) + + defer.returnValue((200, {"event_id": event.event_id})) + + def on_GET(self, request, room_id, event_type, txn_id): + return (200, "Not implemented") + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, event_type, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + + response = yield self.on_POST(request, room_id, event_type) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + + +# TODO: Needs unit testing for room ID + alias joins +class JoinRoomAliasServlet(RestServlet): + + def register(self, http_server): + # /join/$room_identifier[/$txn_id] + PATTERN = ("/join/(?P[^/]*)") + register_txn_path(self, PATTERN, http_server) + + @defer.inlineCallbacks + def on_POST(self, request, room_identifier): + user = yield self.auth.get_user_by_req(request) + + # the identifier could be a room alias or a room id. Try one then the + # other if it fails to parse, without swallowing other valid + # SynapseErrors. + + identifier = None + is_room_alias = False + try: + identifier = self.hs.parse_roomalias(room_identifier) + is_room_alias = True + except SynapseError: + identifier = self.hs.parse_roomid(room_identifier) + + # TODO: Support for specifying the home server to join with? + + if is_room_alias: + handler = self.handlers.room_member_handler + ret_dict = yield handler.join_room_alias(user, identifier) + defer.returnValue((200, ret_dict)) + else: # room id + msg_handler = self.handlers.message_handler + yield msg_handler.create_and_send_event( + { + "type": EventTypes.Member, + "content": {"membership": Membership.JOIN}, + "room_id": identifier.to_string(), + "sender": user.to_string(), + "state_key": user.to_string(), + } + ) + + defer.returnValue((200, {"room_id": identifier.to_string()})) + + @defer.inlineCallbacks + def on_PUT(self, request, room_identifier, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + + response = yield self.on_POST(request, room_identifier) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + + +# TODO: Needs unit testing +class PublicRoomListRestServlet(RestServlet): + PATTERN = client_path_pattern("/publicRooms$") + + @defer.inlineCallbacks + def on_GET(self, request): + handler = self.handlers.room_list_handler + data = yield handler.get_public_room_list() + defer.returnValue((200, data)) + + +# TODO: Needs unit testing +class RoomMemberListRestServlet(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/members$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + # TODO support Pagination stream API (limit/tokens) + user = yield self.auth.get_user_by_req(request) + handler = self.handlers.room_member_handler + members = yield handler.get_room_members_as_pagination_chunk( + room_id=room_id, + user_id=user.to_string()) + + for event in members["chunk"]: + # FIXME: should probably be state_key here, not user_id + target_user = self.hs.parse_userid(event["user_id"]) + # Presence is an optional cache; don't fail if we can't fetch it + try: + presence_handler = self.handlers.presence_handler + presence_state = yield presence_handler.get_state( + target_user=target_user, auth_user=user + ) + event["content"].update(presence_state) + except: + pass + + defer.returnValue((200, members)) + + +# TODO: Needs unit testing +class RoomMessageListRestServlet(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/messages$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + user = yield self.auth.get_user_by_req(request) + pagination_config = PaginationConfig.from_request( + request, default_limit=10, + ) + with_feedback = "feedback" in request.args + as_client_event = "raw" not in request.args + handler = self.handlers.message_handler + msgs = yield handler.get_messages( + room_id=room_id, + user_id=user.to_string(), + pagin_config=pagination_config, + feedback=with_feedback, + as_client_event=as_client_event + ) + + defer.returnValue((200, msgs)) + + +# TODO: Needs unit testing +class RoomStateRestServlet(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/state$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + user = yield self.auth.get_user_by_req(request) + handler = self.handlers.message_handler + # Get all the current state for this room + events = yield handler.get_state_events( + room_id=room_id, + user_id=user.to_string(), + ) + defer.returnValue((200, events)) + + +# TODO: Needs unit testing +class RoomInitialSyncRestServlet(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/initialSync$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + user = yield self.auth.get_user_by_req(request) + pagination_config = PaginationConfig.from_request(request) + content = yield self.handlers.message_handler.room_initial_sync( + room_id=room_id, + user_id=user.to_string(), + pagin_config=pagination_config, + ) + defer.returnValue((200, content)) + + +class RoomTriggerBackfill(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/backfill$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + remote_server = urllib.unquote( + request.args["remote"][0] + ).decode("UTF-8") + + limit = int(request.args["limit"][0]) + + handler = self.handlers.federation_handler + events = yield handler.backfill(remote_server, room_id, limit) + + res = [self.hs.serialize_event(event) for event in events] + defer.returnValue((200, res)) + + +# TODO: Needs unit testing +class RoomMembershipRestServlet(RestServlet): + + def register(self, http_server): + # /rooms/$roomid/[invite|join|leave] + PATTERN = ("/rooms/(?P[^/]*)/" + "(?Pjoin|invite|leave|ban|kick)") + register_txn_path(self, PATTERN, http_server) + + @defer.inlineCallbacks + def on_POST(self, request, room_id, membership_action): + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + # target user is you unless it is an invite + state_key = user.to_string() + if membership_action in ["invite", "ban", "kick"]: + if "user_id" not in content: + raise SynapseError(400, "Missing user_id key.") + state_key = content["user_id"] + + if membership_action == "kick": + membership_action = "leave" + + msg_handler = self.handlers.message_handler + yield msg_handler.create_and_send_event( + { + "type": EventTypes.Member, + "content": {"membership": unicode(membership_action)}, + "room_id": room_id, + "sender": user.to_string(), + "state_key": state_key, + } + ) + + defer.returnValue((200, {})) + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, membership_action, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + + response = yield self.on_POST(request, room_id, membership_action) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + + +class RoomRedactEventRestServlet(RestServlet): + def register(self, http_server): + PATTERN = ("/rooms/(?P[^/]*)/redact/(?P[^/]*)") + register_txn_path(self, PATTERN, http_server) + + @defer.inlineCallbacks + def on_POST(self, request, room_id, event_id): + user = yield self.auth.get_user_by_req(request) + content = _parse_json(request) + + msg_handler = self.handlers.message_handler + event = yield msg_handler.create_and_send_event( + { + "type": EventTypes.Redaction, + "content": content, + "room_id": room_id, + "sender": user.to_string(), + "redacts": event_id, + } + ) + + defer.returnValue((200, {"event_id": event.event_id})) + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, event_id, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + + response = yield self.on_POST(request, room_id, event_id) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + + +class RoomTypingRestServlet(RestServlet): + PATTERN = client_path_pattern( + "/rooms/(?P[^/]*)/typing/(?P[^/]*)$" + ) + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, user_id): + auth_user = yield self.auth.get_user_by_req(request) + + room_id = urllib.unquote(room_id) + target_user = self.hs.parse_userid(urllib.unquote(user_id)) + + content = _parse_json(request) + + typing_handler = self.handlers.typing_notification_handler + + if content["typing"]: + yield typing_handler.started_typing( + target_user=target_user, + auth_user=auth_user, + room_id=room_id, + timeout=content.get("timeout", 30000), + ) + else: + yield typing_handler.stopped_typing( + target_user=target_user, + auth_user=auth_user, + room_id=room_id, + ) + + defer.returnValue((200, {})) + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + + +def register_txn_path(servlet, regex_string, http_server, with_get=False): + """Registers a transaction-based path. + + This registers two paths: + PUT regex_string/$txnid + POST regex_string + + Args: + regex_string (str): The regex string to register. Must NOT have a + trailing $ as this string will be appended to. + http_server : The http_server to register paths with. + with_get: True to also register respective GET paths for the PUTs. + """ + http_server.register_path( + "POST", + client_path_pattern(regex_string + "$"), + servlet.on_POST + ) + http_server.register_path( + "PUT", + client_path_pattern(regex_string + "/(?P[^/]*)$"), + servlet.on_PUT + ) + if with_get: + http_server.register_path( + "GET", + client_path_pattern(regex_string + "/(?P[^/]*)$"), + servlet.on_GET + ) + + +def register_servlets(hs, http_server): + RoomStateEventRestServlet(hs).register(http_server) + RoomCreateRestServlet(hs).register(http_server) + RoomMemberListRestServlet(hs).register(http_server) + RoomMessageListRestServlet(hs).register(http_server) + JoinRoomAliasServlet(hs).register(http_server) + RoomTriggerBackfill(hs).register(http_server) + RoomMembershipRestServlet(hs).register(http_server) + RoomSendEventRestServlet(hs).register(http_server) + PublicRoomListRestServlet(hs).register(http_server) + RoomStateRestServlet(hs).register(http_server) + RoomInitialSyncRestServlet(hs).register(http_server) + RoomRedactEventRestServlet(hs).register(http_server) + RoomTypingRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1/transactions.py b/synapse/rest/client/v1/transactions.py new file mode 100644 index 0000000000..d933fea18a --- /dev/null +++ b/synapse/rest/client/v1/transactions.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains logic for storing HTTP PUT transactions. This is used +to ensure idempotency when performing PUTs using the REST API.""" +import logging + +logger = logging.getLogger(__name__) + + +# FIXME: elsewhere we use FooStore to indicate something in the storage layer... +class HttpTransactionStore(object): + + def __init__(self): + # { key : (txn_id, response) } + self.transactions = {} + + def get_response(self, key, txn_id): + """Retrieve a response for this request. + + Args: + key (str): A transaction-independent key for this request. Usually + this is a combination of the path (without the transaction id) + and the user's access token. + txn_id (str): The transaction ID for this request + Returns: + A tuple of (HTTP response code, response content) or None. + """ + try: + logger.debug("get_response Key: %s TxnId: %s", key, txn_id) + (last_txn_id, response) = self.transactions[key] + if txn_id == last_txn_id: + logger.info("get_response: Returning a response for %s", key) + return response + except KeyError: + pass + return None + + def store_response(self, key, txn_id, response): + """Stores an HTTP response tuple. + + Args: + key (str): A transaction-independent key for this request. Usually + this is a combination of the path (without the transaction id) + and the user's access token. + txn_id (str): The transaction ID for this request. + response (tuple): A tuple of (HTTP response code, response content) + """ + logger.debug("store_response Key: %s TxnId: %s", key, txn_id) + self.transactions[key] = (txn_id, response) + + def store_client_transaction(self, request, txn_id, response): + """Stores the request/response pair of an HTTP transaction. + + Args: + request (twisted.web.http.Request): The twisted HTTP request. This + request must have the transaction ID as the last path segment. + response (tuple): A tuple of (response code, response dict) + txn_id (str): The transaction ID for this request. + """ + self.store_response(self._get_key(request), txn_id, response) + + def get_client_transaction(self, request, txn_id): + """Retrieves a stored response if there was one. + + Args: + request (twisted.web.http.Request): The twisted HTTP request. This + request must have the transaction ID as the last path segment. + txn_id (str): The transaction ID for this request. + Returns: + The response tuple. + Raises: + KeyError if the transaction was not found. + """ + response = self.get_response(self._get_key(request), txn_id) + if response is None: + raise KeyError("Transaction not found.") + return response + + def _get_key(self, request): + token = request.args["access_token"][0] + path_without_txn_id = request.path.rsplit("/", 1)[0] + return path_without_txn_id + "/" + token diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py new file mode 100644 index 0000000000..011c35e69b --- /dev/null +++ b/synapse/rest/client/v1/voip.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from base import RestServlet, client_path_pattern + + +import hmac +import hashlib +import base64 + + +class VoipRestServlet(RestServlet): + PATTERN = client_path_pattern("/voip/turnServer$") + + @defer.inlineCallbacks + def on_GET(self, request): + auth_user = yield self.auth.get_user_by_req(request) + + turnUris = self.hs.config.turn_uris + turnSecret = self.hs.config.turn_shared_secret + userLifetime = self.hs.config.turn_user_lifetime + if not turnUris or not turnSecret or not userLifetime: + defer.returnValue((200, {})) + + expiry = (self.hs.get_clock().time_msec() + userLifetime) / 1000 + username = "%d:%s" % (expiry, auth_user.to_string()) + + mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1) + # We need to use standard base64 encoding here, *not* syutil's + # encode_base64 because we need to add the standard padding to get the + # same result as the TURN server. + password = base64.b64encode(mac.digest()) + + defer.returnValue((200, { + 'username': username, + 'password': password, + 'ttl': userLifetime / 1000, + 'uris': turnUris, + })) + + def on_OPTIONS(self, request): + return (200, {}) + + +def register_servlets(hs, http_server): + VoipRestServlet(hs).register(http_server) diff --git a/synapse/rest/media/__init__.py b/synapse/rest/media/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/rest/media/v0/__init__.py b/synapse/rest/media/v0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py new file mode 100644 index 0000000000..79ae0e3d74 --- /dev/null +++ b/synapse/rest/media/v0/content_repository.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.http.server import respond_with_json_bytes + +from synapse.util.stringutils import random_string +from synapse.api.errors import ( + cs_exception, SynapseError, CodeMessageException, Codes, cs_error +) + +from twisted.protocols.basic import FileSender +from twisted.web import server, resource +from twisted.internet import defer + +import base64 +import json +import logging +import os +import re + +logger = logging.getLogger(__name__) + + +class ContentRepoResource(resource.Resource): + """Provides file uploading and downloading. + + Uploads are POSTed to wherever this Resource is linked to. This resource + returns a "content token" which can be used to GET this content again. The + token is typically a path, but it may not be. Tokens can expire, be + one-time uses, etc. + + In this case, the token is a path to the file and contains 3 interesting + sections: + - User ID base64d (for namespacing content to each user) + - random 24 char string + - Content type base64d (so we can return it when clients GET it) + + """ + isLeaf = True + + def __init__(self, hs, directory, auth, external_addr): + resource.Resource.__init__(self) + self.hs = hs + self.directory = directory + self.auth = auth + self.external_addr = external_addr.rstrip('/') + self.max_upload_size = hs.config.max_upload_size + + if not os.path.isdir(self.directory): + os.mkdir(self.directory) + logger.info("ContentRepoResource : Created %s directory.", + self.directory) + + @defer.inlineCallbacks + def map_request_to_name(self, request): + # auth the user + auth_user = yield self.auth.get_user_by_req(request) + + # namespace all file uploads on the user + prefix = base64.urlsafe_b64encode( + auth_user.to_string() + ).replace('=', '') + + # use a random string for the main portion + main_part = random_string(24) + + # suffix with a file extension if we can make one. This is nice to + # provide a hint to clients on the file information. We will also reuse + # this info to spit back the content type to the client. + suffix = "" + if request.requestHeaders.hasHeader("Content-Type"): + content_type = request.requestHeaders.getRawHeaders( + "Content-Type")[0] + suffix = "." + base64.urlsafe_b64encode(content_type) + if (content_type.split("/")[0].lower() in + ["image", "video", "audio"]): + file_ext = content_type.split("/")[-1] + # be a little paranoid and only allow a-z + file_ext = re.sub("[^a-z]", "", file_ext) + suffix += "." + file_ext + + file_name = prefix + main_part + suffix + file_path = os.path.join(self.directory, file_name) + logger.info("User %s is uploading a file to path %s", + auth_user.to_string(), + file_path) + + # keep trying to make a non-clashing file, with a sensible max attempts + attempts = 0 + while os.path.exists(file_path): + main_part = random_string(24) + file_name = prefix + main_part + suffix + file_path = os.path.join(self.directory, file_name) + attempts += 1 + if attempts > 25: # really? Really? + raise SynapseError(500, "Unable to create file.") + + defer.returnValue(file_path) + + def render_GET(self, request): + # no auth here on purpose, to allow anyone to view, even across home + # servers. + + # TODO: A little crude here, we could do this better. + filename = request.path.split('/')[-1] + # be paranoid + filename = re.sub("[^0-9A-z.-_]", "", filename) + + file_path = self.directory + "/" + filename + + logger.debug("Searching for %s", file_path) + + if os.path.isfile(file_path): + # filename has the content type + base64_contentype = filename.split(".")[1] + content_type = base64.urlsafe_b64decode(base64_contentype) + logger.info("Sending file %s", file_path) + f = open(file_path, 'rb') + request.setHeader('Content-Type', content_type) + + # cache for at least a day. + # XXX: we might want to turn this off for data we don't want to + # recommend caching as it's sensitive or private - or at least + # select private. don't bother setting Expires as all our matrix + # clients are smart enough to be happy with Cache-Control (right?) + request.setHeader( + "Cache-Control", "public,max-age=86400,s-maxage=86400" + ) + + d = FileSender().beginFileTransfer(f, request) + + # after the file has been sent, clean up and finish the request + def cbFinished(ignored): + f.close() + request.finish() + d.addCallback(cbFinished) + else: + respond_with_json_bytes( + request, + 404, + json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)), + send_cors=True) + + return server.NOT_DONE_YET + + def render_POST(self, request): + self._async_render(request) + return server.NOT_DONE_YET + + def render_OPTIONS(self, request): + respond_with_json_bytes(request, 200, {}, send_cors=True) + return server.NOT_DONE_YET + + @defer.inlineCallbacks + def _async_render(self, request): + try: + # TODO: The checks here are a bit late. The content will have + # already been uploaded to a tmp file at this point + content_length = request.getHeader("Content-Length") + if content_length is None: + raise SynapseError( + msg="Request must specify a Content-Length", code=400 + ) + if int(content_length) > self.max_upload_size: + raise SynapseError( + msg="Upload request body is too large", + code=413, + ) + + fname = yield self.map_request_to_name(request) + + # TODO I have a suspicious feeling this is just going to block + with open(fname, "wb") as f: + f.write(request.content.read()) + + # FIXME (erikj): These should use constants. + file_name = os.path.basename(fname) + # FIXME: we can't assume what the repo's public mounted path is + # ...plus self-signed SSL won't work to remote clients anyway + # ...and we can't assume that it's SSL anyway, as we might want to + # serve it via the non-SSL listener... + url = "%s/_matrix/content/%s" % ( + self.external_addr, file_name + ) + + respond_with_json_bytes(request, 200, + json.dumps({"content_token": url}), + send_cors=True) + + except CodeMessageException as e: + logger.exception(e) + respond_with_json_bytes(request, e.code, + json.dumps(cs_exception(e))) + except Exception as e: + logger.error("Failed to store file: %s" % e) + respond_with_json_bytes( + request, + 500, + json.dumps({"error": "Internal server error"}), + send_cors=True) diff --git a/synapse/rest/media/v1/__init__.py b/synapse/rest/media/v1/__init__.py new file mode 100644 index 0000000000..d6c6690577 --- /dev/null +++ b/synapse/rest/media/v1/__init__.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import PIL.Image + +# check for JPEG support. +try: + PIL.Image._getdecoder("rgb", "jpeg", None) +except IOError as e: + if str(e).startswith("decoder jpeg not available"): + raise Exception( + "FATAL: jpeg codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" + " pip install pillow --user'" + ) +except Exception: + # any other exception is fine + pass + + +# check for PNG support. +try: + PIL.Image._getdecoder("rgb", "zip", None) +except IOError as e: + if str(e).startswith("decoder zip not available"): + raise Exception( + "FATAL: zip codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" + " pip install pillow --user'" + ) +except Exception: + # any other exception is fine + pass diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py new file mode 100644 index 0000000000..688e7376ad --- /dev/null +++ b/synapse/rest/media/v1/base_resource.py @@ -0,0 +1,378 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .thumbnailer import Thumbnailer + +from synapse.http.server import respond_with_json +from synapse.util.stringutils import random_string +from synapse.api.errors import ( + cs_exception, CodeMessageException, cs_error, Codes, SynapseError +) + +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.protocols.basic import FileSender + +import os + +import logging + +logger = logging.getLogger(__name__) + + +class BaseMediaResource(Resource): + isLeaf = True + + def __init__(self, hs, filepaths): + Resource.__init__(self) + self.auth = hs.get_auth() + self.client = hs.get_http_client() + self.clock = hs.get_clock() + self.server_name = hs.hostname + self.store = hs.get_datastore() + self.max_upload_size = hs.config.max_upload_size + self.max_image_pixels = hs.config.max_image_pixels + self.filepaths = filepaths + self.downloads = {} + + @staticmethod + def catch_errors(request_handler): + @defer.inlineCallbacks + def wrapped_request_handler(self, request): + try: + yield request_handler(self, request) + except CodeMessageException as e: + logger.exception(e) + respond_with_json( + request, e.code, cs_exception(e), send_cors=True + ) + except: + logger.exception( + "Failed handle request %s.%s on %r", + request_handler.__module__, + request_handler.__name__, + self, + ) + respond_with_json( + request, + 500, + {"error": "Internal server error"}, + send_cors=True + ) + return wrapped_request_handler + + @staticmethod + def _parse_media_id(request): + try: + server_name, media_id = request.postpath + return (server_name, media_id) + except: + raise SynapseError( + 404, + "Invalid media id token %r" % (request.postpath,), + Codes.UNKKOWN, + ) + + @staticmethod + def _parse_integer(request, arg_name, default=None): + try: + if default is None: + return int(request.args[arg_name][0]) + else: + return int(request.args.get(arg_name, [default])[0]) + except: + raise SynapseError( + 400, + "Missing integer argument %r" % (arg_name,), + Codes.UNKNOWN, + ) + + @staticmethod + def _parse_string(request, arg_name, default=None): + try: + if default is None: + return request.args[arg_name][0] + else: + return request.args.get(arg_name, [default])[0] + except: + raise SynapseError( + 400, + "Missing string argument %r" % (arg_name,), + Codes.UNKNOWN, + ) + + def _respond_404(self, request): + respond_with_json( + request, 404, + cs_error( + "Not found %r" % (request.postpath,), + code=Codes.NOT_FOUND, + ), + send_cors=True + ) + + @staticmethod + def _makedirs(filepath): + dirname = os.path.dirname(filepath) + if not os.path.exists(dirname): + os.makedirs(dirname) + + def _get_remote_media(self, server_name, media_id): + key = (server_name, media_id) + download = self.downloads.get(key) + if download is None: + download = self._get_remote_media_impl(server_name, media_id) + self.downloads[key] = download + + @download.addBoth + def callback(media_info): + del self.downloads[key] + return media_info + return download + + @defer.inlineCallbacks + def _get_remote_media_impl(self, server_name, media_id): + media_info = yield self.store.get_cached_remote_media( + server_name, media_id + ) + if not media_info: + media_info = yield self._download_remote_file( + server_name, media_id + ) + defer.returnValue(media_info) + + @defer.inlineCallbacks + def _download_remote_file(self, server_name, media_id): + file_id = random_string(24) + + fname = self.filepaths.remote_media_filepath( + server_name, file_id + ) + self._makedirs(fname) + + try: + with open(fname, "wb") as f: + request_path = "/".join(( + "/_matrix/media/v1/download", server_name, media_id, + )) + length, headers = yield self.client.get_file( + server_name, request_path, output_stream=f, + max_size=self.max_upload_size, + ) + media_type = headers["Content-Type"][0] + time_now_ms = self.clock.time_msec() + + yield self.store.store_cached_remote_media( + origin=server_name, + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=None, + media_length=length, + filesystem_id=file_id, + ) + except: + os.remove(fname) + raise + + media_info = { + "media_type": media_type, + "media_length": length, + "upload_name": None, + "created_ts": time_now_ms, + "filesystem_id": file_id, + } + + yield self._generate_remote_thumbnails( + server_name, media_id, media_info + ) + + defer.returnValue(media_info) + + @defer.inlineCallbacks + def _respond_with_file(self, request, media_type, file_path, + file_size=None): + logger.debug("Responding with %r", file_path) + + if os.path.isfile(file_path): + request.setHeader(b"Content-Type", media_type.encode("UTF-8")) + + # cache for at least a day. + # XXX: we might want to turn this off for data we don't want to + # recommend caching as it's sensitive or private - or at least + # select private. don't bother setting Expires as all our + # clients are smart enough to be happy with Cache-Control + request.setHeader( + b"Cache-Control", b"public,max-age=86400,s-maxage=86400" + ) + if file_size is None: + stat = os.stat(file_path) + file_size = stat.st_size + + request.setHeader( + b"Content-Length", b"%d" % (file_size,) + ) + + with open(file_path, "rb") as f: + yield FileSender().beginFileTransfer(f, request) + + request.finish() + else: + self._respond_404(request) + + def _get_thumbnail_requirements(self, media_type): + if media_type == "image/jpeg": + return ( + (32, 32, "crop", "image/jpeg"), + (96, 96, "crop", "image/jpeg"), + (320, 240, "scale", "image/jpeg"), + (640, 480, "scale", "image/jpeg"), + ) + elif (media_type == "image/png") or (media_type == "image/gif"): + return ( + (32, 32, "crop", "image/png"), + (96, 96, "crop", "image/png"), + (320, 240, "scale", "image/png"), + (640, 480, "scale", "image/png"), + ) + else: + return () + + @defer.inlineCallbacks + def _generate_local_thumbnails(self, media_id, media_info): + media_type = media_info["media_type"] + requirements = self._get_thumbnail_requirements(media_type) + if not requirements: + return + + input_path = self.filepaths.local_media_filepath(media_id) + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, m_height, self.max_image_pixels + ) + return + + scales = set() + crops = set() + for r_width, r_height, r_method, r_type in requirements: + if r_method == "scale": + t_width, t_height = thumbnailer.aspect(r_width, r_height) + scales.add(( + min(m_width, t_width), min(m_height, t_height), r_type, + )) + elif r_method == "crop": + crops.add((r_width, r_height, r_type)) + + for t_width, t_height, t_type in scales: + t_method = "scale" + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + yield self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len + ) + + for t_width, t_height, t_type in crops: + if (t_width, t_height, t_type) in scales: + # If the aspect ratio of the cropped thumbnail matches a purely + # scaled one then there is no point in calculating a separate + # thumbnail. + continue + t_method = "crop" + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + yield self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len + ) + + defer.returnValue({ + "width": m_width, + "height": m_height, + }) + + @defer.inlineCallbacks + def _generate_remote_thumbnails(self, server_name, media_id, media_info): + media_type = media_info["media_type"] + file_id = media_info["filesystem_id"] + requirements = self._get_thumbnail_requirements(media_type) + if not requirements: + return + + input_path = self.filepaths.remote_media_filepath(server_name, file_id) + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, m_height, self.max_image_pixels + ) + return + + scales = set() + crops = set() + for r_width, r_height, r_method, r_type in requirements: + if r_method == "scale": + t_width, t_height = thumbnailer.aspect(r_width, r_height) + scales.add(( + min(m_width, t_width), min(m_height, t_height), r_type, + )) + elif r_method == "crop": + crops.add((r_width, r_height, r_type)) + + for t_width, t_height, t_type in scales: + t_method = "scale" + t_path = self.filepaths.remote_media_thumbnail( + server_name, file_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + yield self.store.store_remote_media_thumbnail( + server_name, media_id, file_id, + t_width, t_height, t_type, t_method, t_len + ) + + for t_width, t_height, t_type in crops: + if (t_width, t_height, t_type) in scales: + # If the aspect ratio of the cropped thumbnail matches a purely + # scaled one then there is no point in calculating a separate + # thumbnail. + continue + t_method = "crop" + t_path = self.filepaths.remote_media_thumbnail( + server_name, file_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + yield self.store.store_remote_media_thumbnail( + server_name, media_id, file_id, + t_width, t_height, t_type, t_method, t_len + ) + + defer.returnValue({ + "width": m_width, + "height": m_height, + }) diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py new file mode 100644 index 0000000000..c585bb11f7 --- /dev/null +++ b/synapse/rest/media/v1/download_resource.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base_resource import BaseMediaResource + +from twisted.web.server import NOT_DONE_YET +from twisted.internet import defer + +import logging + +logger = logging.getLogger(__name__) + + +class DownloadResource(BaseMediaResource): + def render_GET(self, request): + self._async_render_GET(request) + return NOT_DONE_YET + + @BaseMediaResource.catch_errors + @defer.inlineCallbacks + def _async_render_GET(self, request): + try: + server_name, media_id = request.postpath + except: + self._respond_404(request) + return + + if server_name == self.server_name: + yield self._respond_local_file(request, media_id) + else: + yield self._respond_remote_file(request, server_name, media_id) + + @defer.inlineCallbacks + def _respond_local_file(self, request, media_id): + media_info = yield self.store.get_local_media(media_id) + if not media_info: + self._respond_404(request) + return + + media_type = media_info["media_type"] + media_length = media_info["media_length"] + file_path = self.filepaths.local_media_filepath(media_id) + + yield self._respond_with_file( + request, media_type, file_path, media_length + ) + + @defer.inlineCallbacks + def _respond_remote_file(self, request, server_name, media_id): + media_info = yield self._get_remote_media(server_name, media_id) + + media_type = media_info["media_type"] + media_length = media_info["media_length"] + filesystem_id = media_info["filesystem_id"] + + file_path = self.filepaths.remote_media_filepath( + server_name, filesystem_id + ) + + yield self._respond_with_file( + request, media_type, file_path, media_length + ) diff --git a/synapse/rest/media/v1/filepath.py b/synapse/rest/media/v1/filepath.py new file mode 100644 index 0000000000..ed9a58e9d9 --- /dev/null +++ b/synapse/rest/media/v1/filepath.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +class MediaFilePaths(object): + + def __init__(self, base_path): + self.base_path = base_path + + def default_thumbnail(self, default_top_level, default_sub_type, width, + height, content_type, method): + top_level_type, sub_type = content_type.split("/") + file_name = "%i-%i-%s-%s-%s" % ( + width, height, top_level_type, sub_type, method + ) + return os.path.join( + self.base_path, "default_thumbnails", default_top_level, + default_sub_type, file_name + ) + + def local_media_filepath(self, media_id): + return os.path.join( + self.base_path, "local_content", + media_id[0:2], media_id[2:4], media_id[4:] + ) + + def local_media_thumbnail(self, media_id, width, height, content_type, + method): + top_level_type, sub_type = content_type.split("/") + file_name = "%i-%i-%s-%s-%s" % ( + width, height, top_level_type, sub_type, method + ) + return os.path.join( + self.base_path, "local_thumbnails", + media_id[0:2], media_id[2:4], media_id[4:], + file_name + ) + + def remote_media_filepath(self, server_name, file_id): + return os.path.join( + self.base_path, "remote_content", server_name, + file_id[0:2], file_id[2:4], file_id[4:] + ) + + def remote_media_thumbnail(self, server_name, file_id, width, height, + content_type, method): + top_level_type, sub_type = content_type.split("/") + file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type) + return os.path.join( + self.base_path, "remote_thumbnail", server_name, + file_id[0:2], file_id[2:4], file_id[4:], + file_name + ) diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py new file mode 100644 index 0000000000..461cc001f1 --- /dev/null +++ b/synapse/rest/media/v1/media_repository.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .upload_resource import UploadResource +from .download_resource import DownloadResource +from .thumbnail_resource import ThumbnailResource +from .filepath import MediaFilePaths + +from twisted.web.resource import Resource + +import logging + +logger = logging.getLogger(__name__) + + +class MediaRepositoryResource(Resource): + """File uploading and downloading. + + Uploads are POSTed to a resource which returns a token which is used to GET + the download:: + + => POST /_matrix/media/v1/upload HTTP/1.1 + Content-Type: + + + + <= HTTP/1.1 200 OK + Content-Type: application/json + + { "content_uri": "mxc:///" } + + => GET /_matrix/media/v1/download// HTTP/1.1 + + <= HTTP/1.1 200 OK + Content-Type: + Content-Disposition: attachment;filename= + + + + Clients can get thumbnails by supplying a desired width and height and + thumbnailing method:: + + => GET /_matrix/media/v1/thumbnail/ + /?width=&height=&method= HTTP/1.1 + + <= HTTP/1.1 200 OK + Content-Type: image/jpeg or image/png + + + + The thumbnail methods are "crop" and "scale". "scale" trys to return an + image where either the width or the height is smaller than the requested + size. The client should then scale and letterbox the image if it needs to + fit within a given rectangle. "crop" trys to return an image where the + width and height are close to the requested size and the aspect matches + the requested size. The client should scale the image if it needs to fit + within a given rectangle. + """ + + def __init__(self, hs): + Resource.__init__(self) + filepaths = MediaFilePaths(hs.config.media_store_path) + self.putChild("upload", UploadResource(hs, filepaths)) + self.putChild("download", DownloadResource(hs, filepaths)) + self.putChild("thumbnail", ThumbnailResource(hs, filepaths)) diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py new file mode 100644 index 0000000000..84f5e3463c --- /dev/null +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .base_resource import BaseMediaResource + +from twisted.web.server import NOT_DONE_YET +from twisted.internet import defer + +import logging + +logger = logging.getLogger(__name__) + + +class ThumbnailResource(BaseMediaResource): + isLeaf = True + + def render_GET(self, request): + self._async_render_GET(request) + return NOT_DONE_YET + + @BaseMediaResource.catch_errors + @defer.inlineCallbacks + def _async_render_GET(self, request): + server_name, media_id = self._parse_media_id(request) + width = self._parse_integer(request, "width") + height = self._parse_integer(request, "height") + method = self._parse_string(request, "method", "scale") + m_type = self._parse_string(request, "type", "image/png") + + if server_name == self.server_name: + yield self._respond_local_thumbnail( + request, media_id, width, height, method, m_type + ) + else: + yield self._respond_remote_thumbnail( + request, server_name, media_id, + width, height, method, m_type + ) + + @defer.inlineCallbacks + def _respond_local_thumbnail(self, request, media_id, width, height, + method, m_type): + media_info = yield self.store.get_local_media(media_id) + + if not media_info: + self._respond_404(request) + return + + thumbnail_infos = yield self.store.get_local_media_thumbnails(media_id) + + if thumbnail_infos: + thumbnail_info = self._select_thumbnail( + width, height, method, m_type, thumbnail_infos + ) + t_width = thumbnail_info["thumbnail_width"] + t_height = thumbnail_info["thumbnail_height"] + t_type = thumbnail_info["thumbnail_type"] + t_method = thumbnail_info["thumbnail_method"] + + file_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method, + ) + yield self._respond_with_file(request, t_type, file_path) + + else: + yield self._respond_default_thumbnail( + request, media_info, width, height, method, m_type, + ) + + @defer.inlineCallbacks + def _respond_remote_thumbnail(self, request, server_name, media_id, width, + height, method, m_type): + # TODO: Don't download the whole remote file + # We should proxy the thumbnail from the remote server instead. + media_info = yield self._get_remote_media(server_name, media_id) + + thumbnail_infos = yield self.store.get_remote_media_thumbnails( + server_name, media_id, + ) + + if thumbnail_infos: + thumbnail_info = self._select_thumbnail( + width, height, method, m_type, thumbnail_infos + ) + t_width = thumbnail_info["thumbnail_width"] + t_height = thumbnail_info["thumbnail_height"] + t_type = thumbnail_info["thumbnail_type"] + t_method = thumbnail_info["thumbnail_method"] + file_id = thumbnail_info["filesystem_id"] + t_length = thumbnail_info["thumbnail_length"] + + file_path = self.filepaths.remote_media_thumbnail( + server_name, file_id, t_width, t_height, t_type, t_method, + ) + yield self._respond_with_file(request, t_type, file_path, t_length) + else: + yield self._respond_default_thumbnail( + request, media_info, width, height, method, m_type, + ) + + @defer.inlineCallbacks + def _respond_default_thumbnail(self, request, media_info, width, height, + method, m_type): + media_type = media_info["media_type"] + top_level_type = media_type.split("/")[0] + sub_type = media_type.split("/")[-1].split(";")[0] + thumbnail_infos = yield self.store.get_default_thumbnails( + top_level_type, sub_type, + ) + if not thumbnail_infos: + thumbnail_infos = yield self.store.get_default_thumbnails( + top_level_type, "_default", + ) + if not thumbnail_infos: + thumbnail_infos = yield self.store.get_default_thumbnails( + "_default", "_default", + ) + if not thumbnail_infos: + self._respond_404(request) + return + + thumbnail_info = self._select_thumbnail( + width, height, "crop", m_type, thumbnail_infos + ) + + t_width = thumbnail_info["thumbnail_width"] + t_height = thumbnail_info["thumbnail_height"] + t_type = thumbnail_info["thumbnail_type"] + t_method = thumbnail_info["thumbnail_method"] + t_length = thumbnail_info["thumbnail_length"] + + file_path = self.filepaths.default_thumbnail( + top_level_type, sub_type, t_width, t_height, t_type, t_method, + ) + yield self.respond_with_file(request, t_type, file_path, t_length) + + def _select_thumbnail(self, desired_width, desired_height, desired_method, + desired_type, thumbnail_infos): + d_w = desired_width + d_h = desired_height + + if desired_method.lower() == "crop": + info_list = [] + for info in thumbnail_infos: + t_w = info["thumbnail_width"] + t_h = info["thumbnail_height"] + t_method = info["thumbnail_method"] + if t_method == "scale" or t_method == "crop": + aspect_quality = abs(d_w * t_h - d_h * t_w) + size_quality = abs((d_w - t_w) * (d_h - t_h)) + type_quality = desired_type != info["thumbnail_type"] + length_quality = info["thumbnail_length"] + info_list.append(( + aspect_quality, size_quality, type_quality, + length_quality, info + )) + if info_list: + return min(info_list)[-1] + else: + info_list = [] + info_list2 = [] + for info in thumbnail_infos: + t_w = info["thumbnail_width"] + t_h = info["thumbnail_height"] + t_method = info["thumbnail_method"] + size_quality = abs((d_w - t_w) * (d_h - t_h)) + type_quality = desired_type != info["thumbnail_type"] + length_quality = info["thumbnail_length"] + if t_method == "scale" and (t_w >= d_w or t_h >= d_h): + info_list.append(( + size_quality, type_quality, length_quality, info + )) + elif t_method == "scale": + info_list2.append(( + size_quality, type_quality, length_quality, info + )) + if info_list: + return min(info_list)[-1] + else: + return min(info_list2)[-1] diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py new file mode 100644 index 0000000000..28404f2b7b --- /dev/null +++ b/synapse/rest/media/v1/thumbnailer.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import PIL.Image as Image +from io import BytesIO + + +class Thumbnailer(object): + + FORMATS = { + "image/jpeg": "JPEG", + "image/png": "PNG", + } + + def __init__(self, input_path): + self.image = Image.open(input_path) + self.width, self.height = self.image.size + + def aspect(self, max_width, max_height): + """Calculate the largest size that preserves aspect ratio which + fits within the given rectangle:: + + (w_in / h_in) = (w_out / h_out) + w_out = min(w_max, h_max * (w_in / h_in)) + h_out = min(h_max, w_max * (h_in / w_in)) + + Args: + max_width: The largest possible width. + max_height: The larget possible height. + """ + + if max_width * self.height < max_height * self.width: + return (max_width, (max_width * self.height) // self.width) + else: + return ((max_height * self.width) // self.height, max_height) + + def scale(self, output_path, width, height, output_type): + """Rescales the image to the given dimensions""" + scaled = self.image.resize((width, height), Image.ANTIALIAS) + return self.save_image(scaled, output_type, output_path) + + def crop(self, output_path, width, height, output_type): + """Rescales and crops the image to the given dimensions preserving + aspect:: + (w_in / h_in) = (w_scaled / h_scaled) + w_scaled = max(w_out, h_out * (w_in / h_in)) + h_scaled = max(h_out, w_out * (h_in / w_in)) + + Args: + max_width: The largest possible width. + max_height: The larget possible height. + """ + if width * self.height > height * self.width: + scaled_height = (width * self.height) // self.width + scaled_image = self.image.resize( + (width, scaled_height), Image.ANTIALIAS + ) + crop_top = (scaled_height - height) // 2 + crop_bottom = height + crop_top + cropped = scaled_image.crop((0, crop_top, width, crop_bottom)) + else: + scaled_width = (height * self.width) // self.height + scaled_image = self.image.resize( + (scaled_width, height), Image.ANTIALIAS + ) + crop_left = (scaled_width - width) // 2 + crop_right = width + crop_left + cropped = scaled_image.crop((crop_left, 0, crop_right, height)) + return self.save_image(cropped, output_type, output_path) + + def save_image(self, output_image, output_type, output_path): + output_bytes_io = BytesIO() + output_image.save(output_bytes_io, self.FORMATS[output_type], quality=70) + output_bytes = output_bytes_io.getvalue() + with open(output_path, "wb") as output_file: + output_file.write(output_bytes) + return len(output_bytes) diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py new file mode 100644 index 0000000000..b1718a630b --- /dev/null +++ b/synapse/rest/media/v1/upload_resource.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.http.server import respond_with_json + +from synapse.util.stringutils import random_string +from synapse.api.errors import ( + cs_exception, SynapseError, CodeMessageException +) + +from twisted.web.server import NOT_DONE_YET +from twisted.internet import defer + +from .base_resource import BaseMediaResource + +import logging + +logger = logging.getLogger(__name__) + + +class UploadResource(BaseMediaResource): + def render_POST(self, request): + self._async_render_POST(request) + return NOT_DONE_YET + + def render_OPTIONS(self, request): + respond_with_json(request, 200, {}, send_cors=True) + return NOT_DONE_YET + + @defer.inlineCallbacks + def _async_render_POST(self, request): + try: + auth_user = yield self.auth.get_user_by_req(request) + # TODO: The checks here are a bit late. The content will have + # already been uploaded to a tmp file at this point + content_length = request.getHeader("Content-Length") + if content_length is None: + raise SynapseError( + msg="Request must specify a Content-Length", code=400 + ) + if int(content_length) > self.max_upload_size: + raise SynapseError( + msg="Upload request body is too large", + code=413, + ) + + headers = request.requestHeaders + + if headers.hasHeader("Content-Type"): + media_type = headers.getRawHeaders("Content-Type")[0] + else: + raise SynapseError( + msg="Upload request missing 'Content-Type'", + code=400, + ) + + #if headers.hasHeader("Content-Disposition"): + # disposition = headers.getRawHeaders("Content-Disposition")[0] + # TODO(markjh): parse content-dispostion + + media_id = random_string(24) + + fname = self.filepaths.local_media_filepath(media_id) + self._makedirs(fname) + + # This shouldn't block for very long because the content will have + # already been uploaded at this point. + with open(fname, "wb") as f: + f.write(request.content.read()) + + yield self.store.store_local_media( + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=None, + media_length=content_length, + user_id=auth_user, + ) + media_info = { + "media_type": media_type, + "media_length": content_length, + } + + yield self._generate_local_thumbnails(media_id, media_info) + + content_uri = "mxc://%s/%s" % (self.server_name, media_id) + + respond_with_json( + request, 200, {"content_uri": content_uri}, send_cors=True + ) + except CodeMessageException as e: + logger.exception(e) + respond_with_json(request, e.code, cs_exception(e), send_cors=True) + except: + logger.exception("Failed to store file") + respond_with_json( + request, + 500, + {"error": "Internal server error"}, + send_cors=True + ) diff --git a/synapse/server.py b/synapse/server.py index 57a95bf753..e9add8e2b4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -24,7 +24,7 @@ from synapse.events.utils import serialize_event from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers -from synapse.client.v1 import RestServletFactory +from synapse.rest.client.v1 import RestServletFactory from synapse.state import StateHandler from synapse.storage import DataStore from synapse.types import UserID, RoomAlias, RoomID, EventID diff --git a/tests/client/v1/test_events.py b/tests/client/v1/test_events.py index 9b36dd3225..e914b05a52 100644 --- a/tests/client/v1/test_events.py +++ b/tests/client/v1/test_events.py @@ -19,9 +19,9 @@ from tests import unittest # twisted imports from twisted.internet import defer -import synapse.client.v1.events -import synapse.client.v1.register -import synapse.client.v1.room +import synapse.rest.client.v1.events +import synapse.rest.client.v1.register +import synapse.rest.client.v1.room from synapse.server import HomeServer @@ -144,9 +144,9 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_clock().time_msec.return_value = 1000000 hs.get_clock().time.return_value = 1000 - synapse.client.v1.register.register_servlets(hs, self.mock_resource) - synapse.client.v1.events.register_servlets(hs, self.mock_resource) - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.register.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.events.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) # register an account self.user_id = "sid1" diff --git a/tests/client/v1/test_rooms.py b/tests/client/v1/test_rooms.py index 33a8631d76..4d529ef007 100644 --- a/tests/client/v1/test_rooms.py +++ b/tests/client/v1/test_rooms.py @@ -18,7 +18,7 @@ # twisted imports from twisted.internet import defer -import synapse.client.v1.room +import synapse.rest.client.v1.room from synapse.api.constants import Membership from synapse.server import HomeServer @@ -82,7 +82,7 @@ class RoomPermissionsTestCase(RestTestCase): self.auth_user_id = self.rmcreator_id - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) self.auth = hs.get_auth() @@ -476,7 +476,7 @@ class RoomsMemberListTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) def tearDown(self): pass @@ -565,7 +565,7 @@ class RoomsCreateTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) def tearDown(self): pass @@ -668,7 +668,7 @@ class RoomTopicTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) # create the room self.room_id = yield self.create_room_as(self.user_id) @@ -783,7 +783,7 @@ class RoomMemberStateTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) @@ -919,7 +919,7 @@ class RoomMessagesTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) @@ -1023,7 +1023,7 @@ class RoomInitialSyncTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) # Since I'm getting my own presence I need to exist as far as presence # is concerned. diff --git a/tests/client/v1/test_typing.py b/tests/client/v1/test_typing.py index d6d677bde3..af3a9a6c1c 100644 --- a/tests/client/v1/test_typing.py +++ b/tests/client/v1/test_typing.py @@ -18,7 +18,7 @@ # twisted imports from twisted.internet import defer -import synapse.client.v1.room +import synapse.rest.client.v1.room from synapse.server import HomeServer from ...utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey @@ -104,7 +104,7 @@ class RoomTypingTestCase(RestTestCase): hs.get_handlers().room_member_handler.fetch_room_distributions_into = ( fetch_room_distributions_into) - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) # Need another user to make notifications actually work -- cgit 1.4.1 From ca65a9d03e4e6e8bc9b42a3ff5a83c1e5294fc7b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Jan 2015 16:37:08 +0000 Subject: Split out TransactionQueue from replication layer --- synapse/federation/replication.py | 291 +---------------------------- synapse/federation/transaction_queue.py | 314 ++++++++++++++++++++++++++++++++ 2 files changed, 316 insertions(+), 289 deletions(-) create mode 100644 synapse/federation/transaction_queue.py (limited to 'synapse') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 6620532a60..accf95e406 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -20,8 +20,8 @@ a given transport. from twisted.internet import defer from .units import Transaction, Edu - from .persistence import TransactionActions +from .transaction_queue import TransactionQueue from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext @@ -62,7 +62,7 @@ class ReplicationLayer(object): # self.pdu_actions = PduActions(self.store) self.transaction_actions = TransactionActions(self.store) - self._transaction_queue = _TransactionQueue( + self._transaction_queue = TransactionQueue( hs, self.transaction_actions, transport_layer ) @@ -662,290 +662,3 @@ class ReplicationLayer(object): event.internal_metadata.outlier = outlier return event - - -class _TransactionQueue(object): - """This class makes sure we only have one transaction in flight at - a time for a given destination. - - It batches pending PDUs into single transactions. - """ - - def __init__(self, hs, transaction_actions, transport_layer): - self.server_name = hs.hostname - self.transaction_actions = transaction_actions - self.transport_layer = transport_layer - - self._clock = hs.get_clock() - self.store = hs.get_datastore() - - # Is a mapping from destinations -> deferreds. Used to keep track - # of which destinations have transactions in flight and when they are - # done - self.pending_transactions = {} - - # Is a mapping from destination -> list of - # tuple(pending pdus, deferred, order) - self.pending_pdus_by_dest = {} - # destination -> list of tuple(edu, deferred) - self.pending_edus_by_dest = {} - - # destination -> list of tuple(failure, deferred) - self.pending_failures_by_dest = {} - - # HACK to get unique tx id - self._next_txn_id = int(self._clock.time_msec()) - - @defer.inlineCallbacks - @log_function - def enqueue_pdu(self, pdu, destinations, order): - # We loop through all destinations to see whether we already have - # a transaction in progress. If we do, stick it in the pending_pdus - # table and we'll get back to it later. - - destinations = set(destinations) - destinations.discard(self.server_name) - destinations.discard("localhost") - - logger.debug("Sending to: %s", str(destinations)) - - if not destinations: - return - - deferreds = [] - - for destination in destinations: - deferred = defer.Deferred() - self.pending_pdus_by_dest.setdefault(destination, []).append( - (pdu, deferred, order) - ) - - def eb(failure): - if not deferred.called: - deferred.errback(failure) - else: - logger.warn("Failed to send pdu", failure) - - with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) - - deferreds.append(deferred) - - yield defer.DeferredList(deferreds) - - # NO inlineCallbacks - def enqueue_edu(self, edu): - destination = edu.destination - - if destination == self.server_name: - return - - deferred = defer.Deferred() - self.pending_edus_by_dest.setdefault(destination, []).append( - (edu, deferred) - ) - - def eb(failure): - if not deferred.called: - deferred.errback(failure) - else: - logger.warn("Failed to send edu", failure) - - with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) - - return deferred - - @defer.inlineCallbacks - def enqueue_failure(self, failure, destination): - deferred = defer.Deferred() - - self.pending_failures_by_dest.setdefault( - destination, [] - ).append( - (failure, deferred) - ) - - yield deferred - - @defer.inlineCallbacks - @log_function - def _attempt_new_transaction(self, destination): - - (retry_last_ts, retry_interval) = (0, 0) - retry_timings = yield self.store.get_destination_retry_timings( - destination - ) - if retry_timings: - (retry_last_ts, retry_interval) = ( - retry_timings.retry_last_ts, retry_timings.retry_interval - ) - if retry_last_ts + retry_interval > int(self._clock.time_msec()): - logger.info( - "TX [%s] not ready for retry yet - " - "dropping transaction for now", - destination, - ) - return - else: - logger.info("TX [%s] is ready for retry", destination) - - logger.info("TX [%s] _attempt_new_transaction", destination) - - if destination in self.pending_transactions: - # XXX: pending_transactions can get stuck on by a never-ending - # request at which point pending_pdus_by_dest just keeps growing. - # we need application-layer timeouts of some flavour of these - # requests - return - - # list of (pending_pdu, deferred, order) - pending_pdus = self.pending_pdus_by_dest.pop(destination, []) - pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_failures = self.pending_failures_by_dest.pop(destination, []) - - if pending_pdus: - logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", - destination, len(pending_pdus)) - - if not pending_pdus and not pending_edus and not pending_failures: - return - - logger.debug( - "TX [%s] Attempting new transaction" - " (pdus: %d, edus: %d, failures: %d)", - destination, - len(pending_pdus), - len(pending_edus), - len(pending_failures) - ) - - # Sort based on the order field - pending_pdus.sort(key=lambda t: t[2]) - - pdus = [x[0] for x in pending_pdus] - edus = [x[0] for x in pending_edus] - failures = [x[0].get_dict() for x in pending_failures] - deferreds = [ - x[1] - for x in pending_pdus + pending_edus + pending_failures - ] - - try: - self.pending_transactions[destination] = 1 - - logger.debug("TX [%s] Persisting transaction...", destination) - - transaction = Transaction.create_new( - origin_server_ts=int(self._clock.time_msec()), - transaction_id=str(self._next_txn_id), - origin=self.server_name, - destination=destination, - pdus=pdus, - edus=edus, - pdu_failures=failures, - ) - - self._next_txn_id += 1 - - yield self.transaction_actions.prepare_to_send(transaction) - - logger.debug("TX [%s] Persisted transaction", destination) - logger.info( - "TX [%s] Sending transaction [%s]", - destination, - transaction.transaction_id, - ) - - # Actually send the transaction - - # FIXME (erikj): This is a bit of a hack to make the Pdu age - # keys work - def json_data_cb(): - data = transaction.get_dict() - now = int(self._clock.time_msec()) - if "pdus" in data: - for p in data["pdus"]: - if "age_ts" in p: - unsigned = p.setdefault("unsigned", {}) - unsigned["age"] = now - int(p["age_ts"]) - del p["age_ts"] - return data - - code, response = yield self.transport_layer.send_transaction( - transaction, json_data_cb - ) - - logger.info("TX [%s] got %d response", destination, code) - - logger.debug("TX [%s] Sent transaction", destination) - logger.debug("TX [%s] Marking as delivered...", destination) - - yield self.transaction_actions.delivered( - transaction, code, response - ) - - logger.debug("TX [%s] Marked as delivered", destination) - logger.debug("TX [%s] Yielding to callbacks...", destination) - - for deferred in deferreds: - if code == 200: - if retry_last_ts: - # this host is alive! reset retry schedule - yield self.store.set_destination_retry_timings( - destination, 0, 0 - ) - deferred.callback(None) - else: - self.set_retrying(destination, retry_interval) - deferred.errback(RuntimeError("Got status %d" % code)) - - # Ensures we don't continue until all callbacks on that - # deferred have fired - try: - yield deferred - except: - pass - - logger.debug("TX [%s] Yielded to callbacks", destination) - - except Exception as e: - # We capture this here as there as nothing actually listens - # for this finishing functions deferred. - logger.warn( - "TX [%s] Problem in _attempt_transaction: %s", - destination, - e, - ) - - self.set_retrying(destination, retry_interval) - - for deferred in deferreds: - if not deferred.called: - deferred.errback(e) - - finally: - # We want to be *very* sure we delete this after we stop processing - self.pending_transactions.pop(destination, None) - - # Check to see if there is anything else to send. - self._attempt_new_transaction(destination) - - @defer.inlineCallbacks - def set_retrying(self, destination, retry_interval): - # track that this destination is having problems and we should - # give it a chance to recover before trying it again - - if retry_interval: - retry_interval *= 2 - # plateau at hourly retries for now - if retry_interval >= 60 * 60 * 1000: - retry_interval = 60 * 60 * 1000 - else: - retry_interval = 2000 # try again at first after 2 seconds - - yield self.store.set_destination_retry_timings( - destination, - int(self._clock.time_msec()), - retry_interval - ) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py new file mode 100644 index 0000000000..c2cb4a1c49 --- /dev/null +++ b/synapse/federation/transaction_queue.py @@ -0,0 +1,314 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from .units import Transaction + +from synapse.util.logutils import log_function +from synapse.util.logcontext import PreserveLoggingContext + +import logging + + +logger = logging.getLogger(__name__) + + +class TransactionQueue(object): + """This class makes sure we only have one transaction in flight at + a time for a given destination. + + It batches pending PDUs into single transactions. + """ + + def __init__(self, hs, transaction_actions, transport_layer): + self.server_name = hs.hostname + self.transaction_actions = transaction_actions + self.transport_layer = transport_layer + + self._clock = hs.get_clock() + self.store = hs.get_datastore() + + # Is a mapping from destinations -> deferreds. Used to keep track + # of which destinations have transactions in flight and when they are + # done + self.pending_transactions = {} + + # Is a mapping from destination -> list of + # tuple(pending pdus, deferred, order) + self.pending_pdus_by_dest = {} + # destination -> list of tuple(edu, deferred) + self.pending_edus_by_dest = {} + + # destination -> list of tuple(failure, deferred) + self.pending_failures_by_dest = {} + + # HACK to get unique tx id + self._next_txn_id = int(self._clock.time_msec()) + + @defer.inlineCallbacks + @log_function + def enqueue_pdu(self, pdu, destinations, order): + # We loop through all destinations to see whether we already have + # a transaction in progress. If we do, stick it in the pending_pdus + # table and we'll get back to it later. + + destinations = set(destinations) + destinations.discard(self.server_name) + destinations.discard("localhost") + + logger.debug("Sending to: %s", str(destinations)) + + if not destinations: + return + + deferreds = [] + + for destination in destinations: + deferred = defer.Deferred() + self.pending_pdus_by_dest.setdefault(destination, []).append( + (pdu, deferred, order) + ) + + def eb(failure): + if not deferred.called: + deferred.errback(failure) + else: + logger.warn("Failed to send pdu", failure) + + with PreserveLoggingContext(): + self._attempt_new_transaction(destination).addErrback(eb) + + deferreds.append(deferred) + + yield defer.DeferredList(deferreds) + + # NO inlineCallbacks + def enqueue_edu(self, edu): + destination = edu.destination + + if destination == self.server_name: + return + + deferred = defer.Deferred() + self.pending_edus_by_dest.setdefault(destination, []).append( + (edu, deferred) + ) + + def eb(failure): + if not deferred.called: + deferred.errback(failure) + else: + logger.warn("Failed to send edu", failure) + + with PreserveLoggingContext(): + self._attempt_new_transaction(destination).addErrback(eb) + + return deferred + + @defer.inlineCallbacks + def enqueue_failure(self, failure, destination): + deferred = defer.Deferred() + + self.pending_failures_by_dest.setdefault( + destination, [] + ).append( + (failure, deferred) + ) + + yield deferred + + @defer.inlineCallbacks + @log_function + def _attempt_new_transaction(self, destination): + + (retry_last_ts, retry_interval) = (0, 0) + retry_timings = yield self.store.get_destination_retry_timings( + destination + ) + if retry_timings: + (retry_last_ts, retry_interval) = ( + retry_timings.retry_last_ts, retry_timings.retry_interval + ) + if retry_last_ts + retry_interval > int(self._clock.time_msec()): + logger.info( + "TX [%s] not ready for retry yet - " + "dropping transaction for now", + destination, + ) + return + else: + logger.info("TX [%s] is ready for retry", destination) + + logger.info("TX [%s] _attempt_new_transaction", destination) + + if destination in self.pending_transactions: + # XXX: pending_transactions can get stuck on by a never-ending + # request at which point pending_pdus_by_dest just keeps growing. + # we need application-layer timeouts of some flavour of these + # requests + return + + # list of (pending_pdu, deferred, order) + pending_pdus = self.pending_pdus_by_dest.pop(destination, []) + pending_edus = self.pending_edus_by_dest.pop(destination, []) + pending_failures = self.pending_failures_by_dest.pop(destination, []) + + if pending_pdus: + logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", + destination, len(pending_pdus)) + + if not pending_pdus and not pending_edus and not pending_failures: + return + + logger.debug( + "TX [%s] Attempting new transaction" + " (pdus: %d, edus: %d, failures: %d)", + destination, + len(pending_pdus), + len(pending_edus), + len(pending_failures) + ) + + # Sort based on the order field + pending_pdus.sort(key=lambda t: t[2]) + + pdus = [x[0] for x in pending_pdus] + edus = [x[0] for x in pending_edus] + failures = [x[0].get_dict() for x in pending_failures] + deferreds = [ + x[1] + for x in pending_pdus + pending_edus + pending_failures + ] + + try: + self.pending_transactions[destination] = 1 + + logger.debug("TX [%s] Persisting transaction...", destination) + + transaction = Transaction.create_new( + origin_server_ts=int(self._clock.time_msec()), + transaction_id=str(self._next_txn_id), + origin=self.server_name, + destination=destination, + pdus=pdus, + edus=edus, + pdu_failures=failures, + ) + + self._next_txn_id += 1 + + yield self.transaction_actions.prepare_to_send(transaction) + + logger.debug("TX [%s] Persisted transaction", destination) + logger.info( + "TX [%s] Sending transaction [%s]", + destination, + transaction.transaction_id, + ) + + # Actually send the transaction + + # FIXME (erikj): This is a bit of a hack to make the Pdu age + # keys work + def json_data_cb(): + data = transaction.get_dict() + now = int(self._clock.time_msec()) + if "pdus" in data: + for p in data["pdus"]: + if "age_ts" in p: + unsigned = p.setdefault("unsigned", {}) + unsigned["age"] = now - int(p["age_ts"]) + del p["age_ts"] + return data + + code, response = yield self.transport_layer.send_transaction( + transaction, json_data_cb + ) + + logger.info("TX [%s] got %d response", destination, code) + + logger.debug("TX [%s] Sent transaction", destination) + logger.debug("TX [%s] Marking as delivered...", destination) + + yield self.transaction_actions.delivered( + transaction, code, response + ) + + logger.debug("TX [%s] Marked as delivered", destination) + logger.debug("TX [%s] Yielding to callbacks...", destination) + + for deferred in deferreds: + if code == 200: + if retry_last_ts: + # this host is alive! reset retry schedule + yield self.store.set_destination_retry_timings( + destination, 0, 0 + ) + deferred.callback(None) + else: + self.set_retrying(destination, retry_interval) + deferred.errback(RuntimeError("Got status %d" % code)) + + # Ensures we don't continue until all callbacks on that + # deferred have fired + try: + yield deferred + except: + pass + + logger.debug("TX [%s] Yielded to callbacks", destination) + + except Exception as e: + # We capture this here as there as nothing actually listens + # for this finishing functions deferred. + logger.warn( + "TX [%s] Problem in _attempt_transaction: %s", + destination, + e, + ) + + self.set_retrying(destination, retry_interval) + + for deferred in deferreds: + if not deferred.called: + deferred.errback(e) + + finally: + # We want to be *very* sure we delete this after we stop processing + self.pending_transactions.pop(destination, None) + + # Check to see if there is anything else to send. + self._attempt_new_transaction(destination) + + @defer.inlineCallbacks + def set_retrying(self, destination, retry_interval): + # track that this destination is having problems and we should + # give it a chance to recover before trying it again + + if retry_interval: + retry_interval *= 2 + # plateau at hourly retries for now + if retry_interval >= 60 * 60 * 1000: + retry_interval = 60 * 60 * 1000 + else: + retry_interval = 2000 # try again at first after 2 seconds + + yield self.store.set_destination_retry_timings( + destination, + int(self._clock.time_msec()), + retry_interval + ) -- cgit 1.4.1 From dc93860619d56e88844e91f38f66341a32e4c704 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 17:37:12 +0000 Subject: Add rest API & store for creating push rules Also make unrecognised request error look more like synapse errors because it makes it easier to throw them from within rest classes. --- synapse/rest/push_rule.py | 195 ++++++++++++++++++++++++++++++++++++++++++ synapse/storage/push_rule.py | 196 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 391 insertions(+) create mode 100644 synapse/rest/push_rule.py create mode 100644 synapse/storage/push_rule.py (limited to 'synapse') diff --git a/synapse/rest/push_rule.py b/synapse/rest/push_rule.py new file mode 100644 index 0000000000..b5e74479cf --- /dev/null +++ b/synapse/rest/push_rule.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError +from base import RestServlet, client_path_pattern +from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException + +import json + + +class PushRuleRestServlet(RestServlet): + PATTERN = client_path_pattern("/pushrules/.*$") + + def rule_spec_from_path(self, path): + if len(path) < 2: + raise UnrecognizedRequestError() + if path[0] != 'pushrules': + raise UnrecognizedRequestError() + + scope = path[1] + path = path[2:] + if scope not in ['global', 'device']: + raise UnrecognizedRequestError() + + device = None + if scope == 'device': + if len(path) == 0: + raise UnrecognizedRequestError() + device = path[0] + path = path[1:] + + if len(path) == 0: + raise UnrecognizedRequestError() + + template = path[0] + path = path[1:] + + if len(path) == 0: + raise UnrecognizedRequestError() + + rule_id = path[0] + + spec = { + 'scope' : scope, + 'template': template, + 'rule_id': rule_id + } + if device: + spec['device'] = device + return spec + + def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj): + if rule_template in ['override', 'underride']: + if 'conditions' not in req_obj: + raise InvalidRuleException("Missing 'conditions'") + conditions = req_obj['conditions'] + for c in conditions: + if 'kind' not in c: + raise InvalidRuleException("Condition without 'kind'") + elif rule_template == 'room': + conditions = [{ + 'kind': 'event_match', + 'key': 'room_id', + 'pattern': rule_id + }] + elif rule_template == 'sender': + conditions = [{ + 'kind': 'event_match', + 'key': 'user_id', + 'pattern': rule_id + }] + elif rule_template == 'content': + if 'pattern' not in req_obj: + raise InvalidRuleException("Content rule missing 'pattern'") + conditions = [{ + 'kind': 'event_match', + 'key': 'content.body', + 'pattern': req_obj['pattern'] + }] + else: + raise InvalidRuleException("Unknown rule template: %s" % (rule_template)) + + if 'actions' not in req_obj: + raise InvalidRuleException("No actions found") + actions = req_obj['actions'] + + for a in actions: + if a in ['notify', 'dont-notify', 'coalesce']: + pass + elif isinstance(a, dict) and 'set_sound' in a: + pass + else: + raise InvalidRuleException("Unrecognised action") + + return (conditions, actions) + + def priority_class_from_spec(self, spec): + map = { + 'underride': 0, + 'sender': 1, + 'room': 2, + 'content': 3, + 'override': 4 + } + + if spec['template'] not in map.keys(): + raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) + pc = map[spec['template']] + + if spec['scope'] == 'device': + pc += 5 + + return pc + + @defer.inlineCallbacks + def on_PUT(self, request): + spec = self.rule_spec_from_path(request.postpath) + try: + priority_class = self.priority_class_from_spec(spec) + except InvalidRuleException as e: + raise SynapseError(400, e.message) + + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + try: + (conditions, actions) = self.rule_tuple_from_request_object( + spec['template'], + spec['rule_id'], + content + ) + except InvalidRuleException as e: + raise SynapseError(400, e.message) + + before = request.args.get("before", None) + if before and len(before): + before = before[0] + after = request.args.get("after", None) + if after and len(after): + after = after[0] + + try: + yield self.hs.get_datastore().add_push_rule( + user_name=user.to_string(), + rule_id=spec['rule_id'], + priority_class=priority_class, + conditions=conditions, + actions=actions, + before=before, + after=after + ) + except InconsistentRuleException as e: + raise SynapseError(400, e.message) + except RuleNotFoundException: + raise SynapseError(400, "before/after rule not found") + + defer.returnValue((200, {})) + + def on_OPTIONS(self, _): + return 200, {} + + +class InvalidRuleException(Exception): + pass + + +# XXX: C+ped from rest/room.py - surely this should be common? +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + + +def register_servlets(hs, http_server): + PushRuleRestServlet(hs).register(http_server) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py new file mode 100644 index 0000000000..76c4557600 --- /dev/null +++ b/synapse/storage/push_rule.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from ._base import SQLBaseStore, Table +from twisted.internet import defer + +import logging +import copy +import json + +logger = logging.getLogger(__name__) + + +class PushRuleStore(SQLBaseStore): + @defer.inlineCallbacks + def get_push_rules_for_user_name(self, user_name): + sql = ( + "SELECT "+",".join(PushRuleTable.fields)+ + "FROM pushers " + "WHERE user_name = ?" + ) + + rows = yield self._execute(None, sql, user_name) + + dicts = [] + for r in rows: + d = {} + for i, f in enumerate(PushRuleTable.fields): + d[f] = r[i] + dicts.append(d) + + defer.returnValue(dicts) + + @defer.inlineCallbacks + def add_push_rule(self, **kwargs): + vals = copy.copy(kwargs) + if 'conditions' in vals: + vals['conditions'] = json.dumps(vals['conditions']) + if 'actions' in vals: + vals['actions'] = json.dumps(vals['actions']) + # we could check the rest of the keys are valid column names + # but sqlite will do that anyway so I think it's just pointless. + if 'id' in vals: + del vals['id'] + + if 'after' in kwargs or 'before' in kwargs: + ret = yield self.runInteraction( + "_add_push_rule_relative_txn", + self._add_push_rule_relative_txn, + **vals + ) + defer.returnValue(ret) + else: + ret = yield self.runInteraction( + "_add_push_rule_highest_priority_txn", + self._add_push_rule_highest_priority_txn, + **vals + ) + defer.returnValue(ret) + + def _add_push_rule_relative_txn(self, txn, user_name, **kwargs): + after = None + relative_to_rule = None + if 'after' in kwargs and kwargs['after']: + after = kwargs['after'] + relative_to_rule = after + if 'before' in kwargs and kwargs['before']: + relative_to_rule = kwargs['before'] + + # get the priority of the rule we're inserting after/before + sql = ( + "SELECT priority_class, priority FROM "+PushRuleTable.table_name+ + " WHERE user_name = ? and rule_id = ?" + ) + txn.execute(sql, (user_name, relative_to_rule)) + res = txn.fetchall() + if not res: + raise RuleNotFoundException() + (priority_class, base_rule_priority) = res[0] + + if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: + raise InconsistentRuleException( + "Given priority class does not match class of relative rule" + ) + + new_rule = copy.copy(kwargs) + if 'before' in new_rule: + del new_rule['before'] + if 'after' in new_rule: + del new_rule['after'] + new_rule['priority_class'] = priority_class + new_rule['user_name'] = user_name + + # check if the priority before/after is free + new_rule_priority = base_rule_priority + if after: + new_rule_priority -= 1 + else: + new_rule_priority += 1 + + new_rule['priority'] = new_rule_priority + + sql = ( + "SELECT COUNT(*) FROM "+PushRuleTable.table_name+ + " WHERE user_name = ? AND priority_class = ? AND priority = ?" + ) + txn.execute(sql, (user_name, priority_class, new_rule_priority)) + res = txn.fetchall() + num_conflicting = res[0][0] + + # if there are conflicting rules, bump everything + if num_conflicting: + sql = "UPDATE "+PushRuleTable.table_name+" SET priority = priority " + if after: + sql += "-1" + else: + sql += "+1" + sql += " WHERE user_name = ? AND priority_class = ? AND priority " + if after: + sql += "<= ?" + else: + sql += ">= ?" + + txn.execute(sql, (user_name, priority_class, new_rule_priority)) + + # now insert the new rule + sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql += ",".join(new_rule.keys())+") VALUES (" + sql += ", ".join(["?" for _ in new_rule.keys()])+")" + + txn.execute(sql, new_rule.values()) + + def _add_push_rule_highest_priority_txn(self, txn, user_name, priority_class, **kwargs): + # find the highest priority rule in that class + sql = ( + "SELECT COUNT(*), MAX(priority) FROM "+PushRuleTable.table_name+ + " WHERE user_name = ? and priority_class = ?" + ) + txn.execute(sql, (user_name, priority_class)) + res = txn.fetchall() + (how_many, highest_prio) = res[0] + + new_prio = 0 + if how_many > 0: + new_prio = highest_prio + 1 + + # and insert the new rule + new_rule = copy.copy(kwargs) + if 'id' in new_rule: + del new_rule['id'] + new_rule['user_name'] = user_name + new_rule['priority_class'] = priority_class + new_rule['priority'] = new_prio + + sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql += ",".join(new_rule.keys())+") VALUES (" + sql += ", ".join(["?" for _ in new_rule.keys()])+")" + + txn.execute(sql, new_rule.values()) + +class RuleNotFoundException(Exception): + pass + + +class InconsistentRuleException(Exception): + pass + + +class PushRuleTable(Table): + table_name = "push_rules" + + fields = [ + "id", + "user_name", + "rule_id", + "priority_class", + "priority", + "conditions", + "actions", + ] + + EntryType = collections.namedtuple("PushRuleEntry", fields) \ No newline at end of file -- cgit 1.4.1 From ede491b4e0c14d44ce43dd5b152abf148b54b9ed Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 17:38:53 +0000 Subject: Oops: second part of commit dc938606 --- synapse/api/errors.py | 12 ++++++++++++ synapse/http/server.py | 8 ++------ synapse/rest/__init__.py | 3 ++- synapse/storage/__init__.py | 3 +++ synapse/storage/schema/delta/v10.sql | 13 +++++++++++++ synapse/storage/schema/pusher.sql | 13 +++++++++++++ 6 files changed, 45 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index a4155aebae..55181fe77e 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) class Codes(object): + UNRECOGNIZED = "M_UNRECOGNIZED" UNAUTHORIZED = "M_UNAUTHORIZED" FORBIDDEN = "M_FORBIDDEN" BAD_JSON = "M_BAD_JSON" @@ -82,6 +83,17 @@ class RegistrationError(SynapseError): pass +class UnrecognizedRequestError(SynapseError): + """An error indicating we don't understand the request you're trying to make""" + def __init__(self, *args, **kwargs): + if "errcode" not in kwargs: + kwargs["errcode"] = Codes.NOT_FOUND + super(UnrecognizedRequestError, self).__init__( + 400, + "Unrecognized request", + **kwargs + ) + class AuthError(SynapseError): """An error raised when there was a problem authorising an event.""" diff --git a/synapse/http/server.py b/synapse/http/server.py index 8015a22edf..0f6539e1be 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -16,7 +16,7 @@ from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException + cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError ) from synapse.util.logcontext import LoggingContext @@ -139,11 +139,7 @@ class JsonResource(HttpServer, resource.Resource): return # Huh. No one wanted to handle that? Fiiiiiine. Send 400. - self._send_response( - request, - 400, - {"error": "Unrecognized request"} - ) + raise UnrecognizedRequestError() except CodeMessageException as e: if isinstance(e, SynapseError): logger.info("%s SynapseError: %s - %s", request, e.code, e.msg) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 59521d0c77..8e5877cf3f 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -16,7 +16,7 @@ from . import ( room, events, register, login, profile, presence, initial_sync, directory, - voip, admin, pusher, + voip, admin, pusher, push_rule ) @@ -46,3 +46,4 @@ class RestServletFactory(object): voip.register_servlets(hs, client_resource) admin.register_servlets(hs, client_resource) pusher.register_servlets(hs, client_resource) + push_rule.register_servlets(hs, client_resource) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 191fe462a5..11706676d0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,6 +30,7 @@ from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore from .pusher import PusherStore +from .push_rule import PushRuleStore from .media_repository import MediaRepositoryStore from .state import StateStore @@ -62,6 +63,7 @@ SCHEMAS = [ "event_edges", "event_signatures", "pusher", + "push_rules", "media_repository", ] @@ -85,6 +87,7 @@ class DataStore(RoomMemberStore, RoomStore, EventFederationStore, MediaRepositoryStore, PusherStore, + PushRuleStore ): def __init__(self, hs): diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index b84ce20ef3..8c4dfd5c1b 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -31,3 +31,16 @@ CREATE TABLE IF NOT EXISTS pushers ( FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); + +CREATE TABLE IF NOT EXISTS push_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index b84ce20ef3..8c4dfd5c1b 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -31,3 +31,16 @@ CREATE TABLE IF NOT EXISTS pushers ( FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); + +CREATE TABLE IF NOT EXISTS push_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); -- cgit 1.4.1 From 7ecb49ef25937558f1a19a8fe47879d4b9116316 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 17:53:30 +0000 Subject: Insufficient newlines --- synapse/storage/push_rule.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 76c4557600..dbbb35b2ab 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -172,6 +172,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) + class RuleNotFoundException(Exception): pass -- cgit 1.4.1 From 673773b21701c91997512d568bcc8d49a5586b3a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 18:27:07 +0000 Subject: oops, this is not its own schema file --- synapse/storage/__init__.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 11706676d0..8f56d90d95 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -63,7 +63,6 @@ SCHEMAS = [ "event_edges", "event_signatures", "pusher", - "push_rules", "media_repository", ] -- cgit 1.4.1 From 8a850573c9cf50dd83ba47c033b28fe2bbbaf9d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 19:32:17 +0000 Subject: As yet fairly untested GET API for push rules --- synapse/api/errors.py | 14 +++- synapse/rest/client/v1/push_rule.py | 138 +++++++++++++++++++++++++++++++++--- synapse/storage/push_rule.py | 8 +-- 3 files changed, 145 insertions(+), 15 deletions(-) (limited to 'synapse') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 55181fe77e..01207282d6 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -87,13 +87,25 @@ class UnrecognizedRequestError(SynapseError): """An error indicating we don't understand the request you're trying to make""" def __init__(self, *args, **kwargs): if "errcode" not in kwargs: - kwargs["errcode"] = Codes.NOT_FOUND + kwargs["errcode"] = Codes.UNRECOGNIZED super(UnrecognizedRequestError, self).__init__( 400, "Unrecognized request", **kwargs ) + +class NotFoundError(SynapseError): + """An error indicating we can't find the thing you asked for""" + def __init__(self, *args, **kwargs): + if "errcode" not in kwargs: + kwargs["errcode"] = Codes.NOT_FOUND + super(UnrecognizedRequestError, self).__init__( + 404, + "Not found", + **kwargs + ) + class AuthError(SynapseError): """An error raised when there was a problem authorising an event.""" diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index b5e74479cf..2803c1f071 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError +from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError from base import RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException @@ -24,6 +24,14 @@ import json class PushRuleRestServlet(RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") + PRIORITY_CLASS_MAP = { + 'underride': 0, + 'sender': 1, + 'room': 2, + 'content': 3, + 'override': 4 + } + PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} def rule_spec_from_path(self, path): if len(path) < 2: @@ -109,15 +117,7 @@ class PushRuleRestServlet(RestServlet): return (conditions, actions) def priority_class_from_spec(self, spec): - map = { - 'underride': 0, - 'sender': 1, - 'room': 2, - 'content': 3, - 'override': 4 - } - - if spec['template'] not in map.keys(): + if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) pc = map[spec['template']] @@ -171,10 +171,128 @@ class PushRuleRestServlet(RestServlet): defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_GET(self, request): + user = yield self.auth.get_user_by_req(request) + + # we build up the full structure and then decide which bits of it + # to send which means doing unnecessary work sometimes but is + # is probably not going to make a whole lot of difference + rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + + rules = {'global': {}, 'device': {}} + + rules['global'] = _add_empty_priority_class_arrays(rules['global']) + + for r in rawrules: + rulearray = None + + r["conditions"] = json.loads(r["conditions"]) + r["actions"] = json.loads(r["actions"]) + + template_name = _priority_class_to_template_name(r['priority_class']) + + if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: + # per-device rule + instance_handle = _instance_handle_from_conditions(r["conditions"]) + if not instance_handle: + continue + if instance_handle not in rules['device']: + rules['device'][instance_handle] = [] + rules['device'][instance_handle] = \ + _add_empty_priority_class_arrays(rules['device'][instance_handle]) + + rulearray = rules['device'][instance_handle] + else: + rulearray = rules['global'][template_name] + + template_rule = _rule_to_template(r) + if template_rule: + rulearray.append(template_rule) + + path = request.postpath[1:] + if path == []: + defer.returnValue((200, rules)) + + if path[0] == 'global': + path = path[1:] + result = _filter_ruleset_with_path(rules['global'], path) + defer.returnValue((200, result)) + elif path[0] == 'device': + path = path[1:] + if path == []: + raise UnrecognizedRequestError + instance_handle = path[0] + if instance_handle not in rules['device']: + ret = {} + ret = _add_empty_priority_class_arrays(ret) + defer.returnValue((200, ret)) + ruleset = rules['device'][instance_handle] + result = _filter_ruleset_with_path(ruleset, path) + defer.returnValue((200, result)) + else: + raise UnrecognizedRequestError() + + def on_OPTIONS(self, _): return 200, {} +def _add_empty_priority_class_arrays(d): + for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): + d[pc] = [] + return d + +def _instance_handle_from_conditions(conditions): + """ + Given a list of conditions, return the instance handle of the + device rule if there is one + """ + for c in conditions: + if c['kind'] == 'device': + return c['instance_handle'] + return None + +def _filter_ruleset_with_path(ruleset, path): + if path == []: + return ruleset + template_kind = path[0] + if template_kind not in ruleset: + raise UnrecognizedRequestError() + path = path[1:] + if path == []: + return ruleset[template_kind] + rule_id = path[0] + for r in ruleset[template_kind]: + if r['rule_id'] == rule_id: + return r + raise NotFoundError + +def _priority_class_to_template_name(pc): + if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: + # per-device + prio_class_index = pc - PushRuleRestServlet.PRIORITY_CLASS_MAP['override'] + return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[prio_class_index] + else: + return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + +def _rule_to_template(rule): + template_name = _priority_class_to_template_name(rule['priority_class']) + if template_name in ['override', 'underride']: + return {k:rule[k] for k in ["rule_id", "conditions", "actions"]} + elif template_name in ["sender", "room"]: + return {k:rule[k] for k in ["rule_id", "actions"]} + elif template_name == 'content': + if len(rule["conditions"]) != 1: + return None + thecond = rule["conditions"][0] + if "pattern" not in thecond: + return None + ret = {k:rule[k] for k in ["rule_id", "actions"]} + ret["pattern"] = thecond["pattern"] + return ret + + class InvalidRuleException(Exception): pass diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index dbbb35b2ab..d087257ffc 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -29,11 +29,11 @@ class PushRuleStore(SQLBaseStore): @defer.inlineCallbacks def get_push_rules_for_user_name(self, user_name): sql = ( - "SELECT "+",".join(PushRuleTable.fields)+ - "FROM pushers " - "WHERE user_name = ?" + "SELECT "+",".join(PushRuleTable.fields)+" " + "FROM "+PushRuleTable.table_name+" " + "WHERE user_name = ? " + "ORDER BY priority_class DESC, priority DESC" ) - rows = yield self._execute(None, sql, user_name) dicts = [] -- cgit 1.4.1 From 6927b6b19783d7134eba3461c4fafe4efdec40f1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 10:21:47 +0000 Subject: This really serves me right for ever making a map called 'map'. --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2803c1f071..7df3fc7f09 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -119,7 +119,7 @@ class PushRuleRestServlet(RestServlet): def priority_class_from_spec(self, spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) - pc = map[spec['template']] + pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] if spec['scope'] == 'device': pc += 5 -- cgit 1.4.1 From bcd48b9636071543fa64e7fb066275d1c9c1e363 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 10:28:25 +0000 Subject: Fix adding rules without before/after & add the rule that we couldn't find to the error --- synapse/rest/client/v1/push_rule.py | 4 ++-- synapse/storage/push_rule.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 7df3fc7f09..77a0772479 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -166,8 +166,8 @@ class PushRuleRestServlet(RestServlet): ) except InconsistentRuleException as e: raise SynapseError(400, e.message) - except RuleNotFoundException: - raise SynapseError(400, "before/after rule not found") + except RuleNotFoundException as e: + raise SynapseError(400, e.message) defer.returnValue((200, {})) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index d087257ffc..2366090e09 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -46,7 +46,7 @@ class PushRuleStore(SQLBaseStore): defer.returnValue(dicts) @defer.inlineCallbacks - def add_push_rule(self, **kwargs): + def add_push_rule(self, before, after, **kwargs): vals = copy.copy(kwargs) if 'conditions' in vals: vals['conditions'] = json.dumps(vals['conditions']) @@ -57,10 +57,12 @@ class PushRuleStore(SQLBaseStore): if 'id' in vals: del vals['id'] - if 'after' in kwargs or 'before' in kwargs: + if before or after: ret = yield self.runInteraction( "_add_push_rule_relative_txn", self._add_push_rule_relative_txn, + before=before, + after=after, **vals ) defer.returnValue(ret) @@ -89,7 +91,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, relative_to_rule)) res = txn.fetchall() if not res: - raise RuleNotFoundException() + raise RuleNotFoundException("before/after rule not found: %s" % (relative_to_rule)) (priority_class, base_rule_priority) = res[0] if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: -- cgit 1.4.1 From f87586e661101849a90f9d106b207a529e4cf689 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 10:32:40 +0000 Subject: right super() param --- synapse/api/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 01207282d6..4f59e1742c 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -100,7 +100,7 @@ class NotFoundError(SynapseError): def __init__(self, *args, **kwargs): if "errcode" not in kwargs: kwargs["errcode"] = Codes.NOT_FOUND - super(UnrecognizedRequestError, self).__init__( + super(NotFoundError, self).__init__( 404, "Not found", **kwargs -- cgit 1.4.1 From 7256def8e43bf5ab982cb7e785fb1334a1ef4ab8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 10:37:38 +0000 Subject: Merge rest servlets into the client json resource object --- synapse/app/homeserver.py | 5 ++--- synapse/rest/client/v1/__init__.py | 18 +++++++----------- synapse/server.py | 10 ---------- tests/rest/client/v1/test_presence.py | 9 ++++++--- tests/rest/client/v1/test_profile.py | 4 +++- 5 files changed, 18 insertions(+), 28 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index cd24bbdc79..fabe8ddacb 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -37,6 +37,7 @@ from synapse.api.urls import ( from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory from synapse.util.logcontext import LoggingContext +from synapse.rest.client.v1 import ClientV1RestResource from daemonize import Daemonize import twisted.manhole.telnet @@ -59,7 +60,7 @@ class SynapseHomeServer(HomeServer): return MatrixFederationHttpClient(self) def build_resource_for_client(self): - return JsonResource() + return ClientV1RestResource(self) def build_resource_for_federation(self): return JsonResource() @@ -224,8 +225,6 @@ def setup(): content_addr=config.content_addr, ) - hs.register_servlets() - hs.create_resource_tree( web_client=config.webclient, redirect_root_to_web_client=True, diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index 88ec9cd27d..8bb89b2f6a 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -19,22 +19,18 @@ from . import ( voip, admin, ) +from synapse.http.server import JsonResource -class RestServletFactory(object): - """ A factory for creating REST servlets. - - These REST servlets represent the entire client-server REST API. Generally - speaking, they serve as wrappers around events and the handlers that - process them. - - See synapse.events for information on synapse events. - """ +class ClientV1RestResource(JsonResource): + """A resource for version 1 of the matrix client API.""" def __init__(self, hs): - client_resource = hs.get_resource_for_client() + JsonResource.__init__(self) + self.register_servlets(self, hs) - # TODO(erikj): There *must* be a better way of doing this. + @staticmethod + def register_servlets(client_resource, hs): room.register_servlets(hs, client_resource) events.register_servlets(hs, client_resource) register.register_servlets(hs, client_resource) diff --git a/synapse/server.py b/synapse/server.py index e9add8e2b4..476d809374 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -24,7 +24,6 @@ from synapse.events.utils import serialize_event from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers -from synapse.rest.client.v1 import RestServletFactory from synapse.state import StateHandler from synapse.storage import DataStore from synapse.types import UserID, RoomAlias, RoomID, EventID @@ -203,9 +202,6 @@ class HomeServer(BaseHomeServer): def build_auth(self): return Auth(self) - def build_rest_servlet_factory(self): - return RestServletFactory(self) - def build_state_handler(self): return StateHandler(self) @@ -229,9 +225,3 @@ class HomeServer(BaseHomeServer): clock=self.get_clock(), hostname=self.hostname, ) - - def register_servlets(self): - """ Register all servlets associated with this HomeServer. - """ - # Simply building the ServletFactory is sufficient to have it register - self.get_rest_servlet_factory() diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 0b6f7cfccb..783720ac29 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -25,6 +25,8 @@ from ....utils import MockHttpResource, MockKey from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler from synapse.server import HomeServer +from synapse.rest.client.v1 import presence +from synapse.rest.client.v1 import events OFFLINE = PresenceState.OFFLINE @@ -86,7 +88,7 @@ class PresenceStateTestCase(unittest.TestCase): return defer.succeed([]) room_member_handler.get_rooms_for_user = get_rooms_for_user - hs.register_servlets() + presence.register_servlets(hs, self.mock_resource) self.u_apple = hs.parse_userid(myid) @@ -172,7 +174,7 @@ class PresenceListTestCase(unittest.TestCase): hs.get_auth().get_user_by_token = _get_user_by_token - hs.register_servlets() + presence.register_servlets(hs, self.mock_resource) self.u_apple = hs.parse_userid("@apple:test") self.u_banana = hs.parse_userid("@banana:test") @@ -283,7 +285,8 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.get_auth().get_user_by_req = _get_user_by_req - hs.register_servlets() + presence.register_servlets(hs, self.mock_resource) + events.register_servlets(hs, self.mock_resource) hs.handlers.room_member_handler = Mock(spec=[]) diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 47cfb10a6d..5b5c3edc22 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -25,6 +25,8 @@ from ....utils import MockHttpResource, MockKey from synapse.api.errors import SynapseError, AuthError from synapse.server import HomeServer +from synapse.rest.client.v1 import profile + myid = "@1234ABCD:test" PATH_PREFIX = "/_matrix/client/api/v1" @@ -61,7 +63,7 @@ class ProfileTestCase(unittest.TestCase): hs.get_handlers().profile_handler = self.mock_handler - hs.register_servlets() + profile.register_servlets(hs, self.mock_resource) @defer.inlineCallbacks def test_get_my_name(self): -- cgit 1.4.1 From 49fe31792bc0cf709248e592baefb8f34606236a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 11:19:02 +0000 Subject: Add slightly pedantic trailing slash error. --- synapse/api/errors.py | 7 ++++++- synapse/rest/client/v1/push_rule.py | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 4f59e1742c..5872e82d0f 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -88,9 +88,14 @@ class UnrecognizedRequestError(SynapseError): def __init__(self, *args, **kwargs): if "errcode" not in kwargs: kwargs["errcode"] = Codes.UNRECOGNIZED + message = None + if len(args) == 0: + message = "Unrecognized request" + else: + message = args[0] super(UnrecognizedRequestError, self).__init__( 400, - "Unrecognized request", + message, **kwargs ) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 77a0772479..6f108431b2 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -32,6 +32,8 @@ class PushRuleRestServlet(RestServlet): 'override': 4 } PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} + SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR =\ + "Unrecognised request: You probably wanted a trailing slash" def rule_spec_from_path(self, path): if len(path) < 2: @@ -211,10 +213,14 @@ class PushRuleRestServlet(RestServlet): rulearray.append(template_rule) path = request.postpath[1:] + if path == []: - defer.returnValue((200, rules)) + # we're a reference impl: pedantry is our job. + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) - if path[0] == 'global': + if path[0] == '': + defer.returnValue((200, rules)) + elif path[0] == 'global': path = path[1:] result = _filter_ruleset_with_path(rules['global'], path) defer.returnValue((200, result)) @@ -255,12 +261,17 @@ def _instance_handle_from_conditions(conditions): def _filter_ruleset_with_path(ruleset, path): if path == []: + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + + if path[0] == '': return ruleset template_kind = path[0] if template_kind not in ruleset: raise UnrecognizedRequestError() path = path[1:] if path == []: + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + if path[0] == '': return ruleset[template_kind] rule_id = path[0] for r in ruleset[template_kind]: -- cgit 1.4.1 From 5759bec43cb52862a8d455afb8cd9d1c5660bc3d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 11:47:15 +0000 Subject: Replace hs.parse_userid with UserID.from_string --- synapse/api/auth.py | 9 +++++---- synapse/handlers/_base.py | 5 +++-- synapse/handlers/events.py | 3 ++- synapse/handlers/federation.py | 13 +++++++------ synapse/handlers/message.py | 13 +++++++------ synapse/handlers/presence.py | 23 ++++++++++++----------- synapse/handlers/profile.py | 3 ++- synapse/handlers/room.py | 14 +++++++------- synapse/handlers/typing.py | 3 ++- synapse/rest/client/v1/admin.py | 4 +++- synapse/rest/client/v1/presence.py | 15 ++++++++------- synapse/rest/client/v1/profile.py | 13 +++++++------ synapse/rest/client/v1/room.py | 5 +++-- synapse/server.py | 6 ------ synapse/storage/roommember.py | 5 +++-- tests/handlers/test_presence.py | 29 +++++++++++++++-------------- tests/handlers/test_presencelike.py | 9 +++++---- tests/handlers/test_profile.py | 8 ++++---- tests/handlers/test_room.py | 9 +++++---- tests/handlers/test_typing.py | 7 ++++--- tests/rest/client/v1/test_presence.py | 19 ++++++++++--------- tests/rest/client/v1/test_profile.py | 3 ++- tests/rest/client/v1/test_rooms.py | 21 +++++++++------------ tests/rest/client/v1/test_typing.py | 5 +++-- tests/storage/test_presence.py | 5 +++-- tests/storage/test_profile.py | 3 ++- tests/storage/test_redaction.py | 5 +++-- tests/storage/test_room.py | 3 ++- tests/storage/test_roommember.py | 7 ++++--- tests/storage/test_stream.py | 5 +++-- tests/test_types.py | 6 ------ 31 files changed, 145 insertions(+), 133 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e31482cfaa..a342a0e0da 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,6 +21,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor +from synapse.types import UserID import logging @@ -104,7 +105,7 @@ class Auth(object): for event in curr_state: if event.type == EventTypes.Member: try: - if self.hs.parse_userid(event.state_key).domain != host: + if UserID.from_string(event.state_key).domain != host: continue except: logger.warn("state_key not user_id: %s", event.state_key) @@ -337,7 +338,7 @@ class Auth(object): user_info = { "admin": bool(ret.get("admin", False)), "device_id": ret.get("device_id"), - "user": self.hs.parse_userid(ret.get("name")), + "user": UserID.from_string(ret.get("name")), } defer.returnValue(user_info) @@ -461,7 +462,7 @@ class Auth(object): "You are not allowed to set others state" ) else: - sender_domain = self.hs.parse_userid( + sender_domain = UserID.from_string( event.user_id ).domain @@ -496,7 +497,7 @@ class Auth(object): # Validate users for k, v in user_list.items(): try: - self.hs.parse_userid(k) + UserID.from_string(k) except: raise SynapseError(400, "Not a valid user_id: %s" % (k,)) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index f33d17a31e..1773fa20aa 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -19,6 +19,7 @@ from synapse.api.errors import LimitExceededError, SynapseError from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes +from synapse.types import UserID import logging @@ -113,7 +114,7 @@ class BaseHandler(object): if event.type == EventTypes.Member: if event.content["membership"] == Membership.INVITE: - invitee = self.hs.parse_userid(event.state_key) + invitee = UserID.from_string(event.state_key) if not self.hs.is_mine(invitee): # TODO: Can we add signature from remote server in a nicer # way? If we have been invited by a remote server, we need @@ -134,7 +135,7 @@ class BaseHandler(object): if k[0] == EventTypes.Member: if s.content["membership"] == Membership.JOIN: destinations.add( - self.hs.parse_userid(s.state_key).domain + UserID.from_string(s.state_key).domain ) except SynapseError: logger.warn( diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 103bc67c42..01e67b0818 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -17,6 +17,7 @@ from twisted.internet import defer from synapse.util.logcontext import PreserveLoggingContext from synapse.util.logutils import log_function +from synapse.types import UserID from ._base import BaseHandler @@ -48,7 +49,7 @@ class EventStreamHandler(BaseHandler): @log_function def get_stream(self, auth_user_id, pagin_config, timeout=0, as_client_event=True): - auth_user = self.hs.parse_userid(auth_user_id) + auth_user = UserID.from_string(auth_user_id) try: if auth_user not in self._streams_per_user: diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 81203bf1a3..bcdcc90a18 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -28,6 +28,7 @@ from synapse.crypto.event_signing import ( compute_event_signature, check_event_content_hash, add_hashes_and_signatures, ) +from synapse.types import UserID from syutil.jsonutil import encode_canonical_json from twisted.internet import defer @@ -227,7 +228,7 @@ class FederationHandler(BaseHandler): extra_users = [] if event.type == EventTypes.Member: target_user_id = event.state_key - target_user = self.hs.parse_userid(target_user_id) + target_user = UserID.from_string(target_user_id) extra_users.append(target_user) yield self.notifier.on_new_room_event( @@ -236,7 +237,7 @@ class FederationHandler(BaseHandler): if event.type == EventTypes.Member: if event.membership == Membership.JOIN: - user = self.hs.parse_userid(event.state_key) + user = UserID.from_string(event.state_key) yield self.distributor.fire( "user_joined_room", user=user, room_id=event.room_id ) @@ -491,7 +492,7 @@ class FederationHandler(BaseHandler): extra_users = [] if event.type == EventTypes.Member: target_user_id = event.state_key - target_user = self.hs.parse_userid(target_user_id) + target_user = UserID.from_string(target_user_id) extra_users.append(target_user) yield self.notifier.on_new_room_event( @@ -500,7 +501,7 @@ class FederationHandler(BaseHandler): if event.type == EventTypes.Member: if event.content["membership"] == Membership.JOIN: - user = self.hs.parse_userid(event.state_key) + user = UserID.from_string(event.state_key) yield self.distributor.fire( "user_joined_room", user=user, room_id=event.room_id ) @@ -514,7 +515,7 @@ class FederationHandler(BaseHandler): if k[0] == EventTypes.Member: if s.content["membership"] == Membership.JOIN: destinations.add( - self.hs.parse_userid(s.state_key).domain + UserID.from_string(s.state_key).domain ) except: logger.warn( @@ -565,7 +566,7 @@ class FederationHandler(BaseHandler): backfilled=False, ) - target_user = self.hs.parse_userid(event.state_key) + target_user = UserID.from_string(event.state_key) yield self.notifier.on_new_room_event( event, extra_users=[target_user], ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index f2a2f16933..6a1104a890 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -20,6 +20,7 @@ from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig from synapse.events.validator import EventValidator from synapse.util.logcontext import PreserveLoggingContext +from synapse.types import UserID from ._base import BaseHandler @@ -89,7 +90,7 @@ class MessageHandler(BaseHandler): yield self.hs.get_event_sources().get_current_token() ) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) events, next_key = yield data_source.get_pagination_rows( user, pagin_config.get_source_config("room"), room_id @@ -130,13 +131,13 @@ class MessageHandler(BaseHandler): if ratelimit: self.ratelimit(builder.user_id) # TODO(paul): Why does 'event' not have a 'user' object? - user = self.hs.parse_userid(builder.user_id) + user = UserID.from_string(builder.user_id) assert self.hs.is_mine(user), "User must be our own: %s" % (user,) if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) if membership == Membership.JOIN: - joinee = self.hs.parse_userid(builder.state_key) + joinee = UserID.from_string(builder.state_key) # If event doesn't include a display name, add one. yield self.distributor.fire( "collect_presencelike_data", @@ -237,7 +238,7 @@ class MessageHandler(BaseHandler): membership_list=[Membership.INVITE, Membership.JOIN] ) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) rooms_ret = [] @@ -316,7 +317,7 @@ class MessageHandler(BaseHandler): # TODO(paul): I wish I was called with user objects not user_id # strings... - auth_user = self.hs.parse_userid(user_id) + auth_user = UserID.from_string(user_id) # TODO: These concurrently state_tuples = yield self.state_handler.get_current_state(room_id) @@ -349,7 +350,7 @@ class MessageHandler(BaseHandler): for m in room_members: try: member_presence = yield presence_handler.get_state( - target_user=self.hs.parse_userid(m.user_id), + target_user=UserID.from_string(m.user_id), auth_user=auth_user, as_event=True, ) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 8aeed99274..d66bfea7b1 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -20,6 +20,7 @@ from synapse.api.constants import PresenceState from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext +from synapse.types import UserID from ._base import BaseHandler @@ -96,22 +97,22 @@ class PresenceHandler(BaseHandler): self.federation.register_edu_handler( "m.presence_invite", lambda origin, content: self.invite_presence( - observed_user=hs.parse_userid(content["observed_user"]), - observer_user=hs.parse_userid(content["observer_user"]), + observed_user=UserID.from_string(content["observed_user"]), + observer_user=UserID.from_string(content["observer_user"]), ) ) self.federation.register_edu_handler( "m.presence_accept", lambda origin, content: self.accept_presence( - observed_user=hs.parse_userid(content["observed_user"]), - observer_user=hs.parse_userid(content["observer_user"]), + observed_user=UserID.from_string(content["observed_user"]), + observer_user=UserID.from_string(content["observer_user"]), ) ) self.federation.register_edu_handler( "m.presence_deny", lambda origin, content: self.deny_presence( - observed_user=hs.parse_userid(content["observed_user"]), - observer_user=hs.parse_userid(content["observer_user"]), + observed_user=UserID.from_string(content["observed_user"]), + observer_user=UserID.from_string(content["observer_user"]), ) ) @@ -418,7 +419,7 @@ class PresenceHandler(BaseHandler): ) for p in presence: - observed_user = self.hs.parse_userid(p.pop("observed_user_id")) + observed_user = UserID.from_string(p.pop("observed_user_id")) p["observed_user"] = observed_user p.update(self._get_or_offline_usercache(observed_user).get_state()) if "last_active" in p: @@ -441,7 +442,7 @@ class PresenceHandler(BaseHandler): user.localpart, accepted=True ) target_users = set([ - self.hs.parse_userid(x["observed_user_id"]) for x in presence + UserID.from_string(x["observed_user_id"]) for x in presence ]) # Also include people in all my rooms @@ -646,7 +647,7 @@ class PresenceHandler(BaseHandler): deferreds = [] for push in content.get("push", []): - user = self.hs.parse_userid(push["user_id"]) + user = UserID.from_string(push["user_id"]) logger.debug("Incoming presence update from %s", user) @@ -694,7 +695,7 @@ class PresenceHandler(BaseHandler): del self._user_cachemap[user] for poll in content.get("poll", []): - user = self.hs.parse_userid(poll) + user = UserID.from_string(poll) if not self.hs.is_mine(user): continue @@ -709,7 +710,7 @@ class PresenceHandler(BaseHandler): deferreds.append(self._push_presence_remote(user, origin)) for unpoll in content.get("unpoll", []): - user = self.hs.parse_userid(unpoll) + user = UserID.from_string(unpoll) if not self.hs.is_mine(user): continue diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 7777d3cc94..03b2159c53 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException from synapse.api.constants import EventTypes, Membership from synapse.util.logcontext import PreserveLoggingContext +from synapse.types import UserID from ._base import BaseHandler @@ -169,7 +170,7 @@ class ProfileHandler(BaseHandler): @defer.inlineCallbacks def on_profile_query(self, args): - user = self.hs.parse_userid(args["user_id"]) + user = UserID.from_string(args["user_id"]) if not self.hs.is_mine(user): raise SynapseError(400, "User is not hosted on this Home Server") diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6d0db18e51..0242288c4e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -64,7 +64,7 @@ class RoomCreationHandler(BaseHandler): invite_list = config.get("invite", []) for i in invite_list: try: - self.hs.parse_userid(i) + UserID.from_string(i) except: raise SynapseError(400, "Invalid user_id: %s" % (i,)) @@ -114,7 +114,7 @@ class RoomCreationHandler(BaseHandler): servers=[self.hs.hostname], ) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) creation_events = self._create_events_for_new_room( user, room_id, is_public=is_public ) @@ -250,7 +250,7 @@ class RoomMemberHandler(BaseHandler): users = yield self.store.get_users_in_room(room_id) - defer.returnValue([hs.parse_userid(u) for u in users]) + defer.returnValue([UserID.from_string(u) for u in users]) @defer.inlineCallbacks def fetch_room_distributions_into(self, room_id, localusers=None, @@ -368,7 +368,7 @@ class RoomMemberHandler(BaseHandler): ) if prev_state and prev_state.membership == Membership.JOIN: - user = self.hs.parse_userid(event.user_id) + user = UserID.from_string(event.user_id) self.distributor.fire( "user_left_room", user=user, room_id=event.room_id ) @@ -412,7 +412,7 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def _do_join(self, event, context, room_host=None, do_auth=True): - joinee = self.hs.parse_userid(event.state_key) + joinee = UserID.from_string(event.state_key) # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id @@ -476,7 +476,7 @@ class RoomMemberHandler(BaseHandler): do_auth=do_auth, ) - user = self.hs.parse_userid(event.user_id) + user = UserID.from_string(event.user_id) yield self.distributor.fire( "user_joined_room", user=user, room_id=room_id ) @@ -526,7 +526,7 @@ class RoomMemberHandler(BaseHandler): do_auth): yield run_on_reactor() - target_user = self.hs.parse_userid(event.state_key) + target_user = UserID.from_string(event.state_key) yield self.handle_new_client_event( event, diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index cd9638dd04..c69787005f 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError, AuthError +from synapse.types import UserID import logging @@ -185,7 +186,7 @@ class TypingNotificationHandler(BaseHandler): @defer.inlineCallbacks def _recv_edu(self, origin, content): room_id = content["room_id"] - user = self.homeserver.parse_userid(content["user_id"]) + user = UserID.from_string(content["user_id"]) localusers = set() diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 0aa83514c8..4aefb94053 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -16,6 +16,8 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError +from synapse.types import UserID + from base import RestServlet, client_path_pattern import logging @@ -28,7 +30,7 @@ class WhoisRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - target_user = self.hs.parse_userid(user_id) + target_user = UserID.from_string(user_id) auth_user = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index ca4d2d21f0..22fcb7d7d0 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -18,7 +18,8 @@ from twisted.internet import defer from synapse.api.errors import SynapseError -from base import RestServlet, client_path_pattern +from synapse.types import UserID +from .base import RestServlet, client_path_pattern import json import logging @@ -32,7 +33,7 @@ class PresenceStatusRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( target_user=user, auth_user=auth_user) @@ -42,7 +43,7 @@ class PresenceStatusRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) state = {} try: @@ -77,7 +78,7 @@ class PresenceListRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) if not self.hs.is_mine(user): raise SynapseError(400, "User not hosted on this Home Server") @@ -97,7 +98,7 @@ class PresenceListRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) if not self.hs.is_mine(user): raise SynapseError(400, "User not hosted on this Home Server") @@ -118,7 +119,7 @@ class PresenceListRestServlet(RestServlet): raise SynapseError(400, "Bad invite value.") if len(u) == 0: continue - invited_user = self.hs.parse_userid(u) + invited_user = UserID.from_string(u) yield self.handlers.presence_handler.send_invite( observer_user=user, observed_user=invited_user ) @@ -129,7 +130,7 @@ class PresenceListRestServlet(RestServlet): raise SynapseError(400, "Bad drop value.") if len(u) == 0: continue - dropped_user = self.hs.parse_userid(u) + dropped_user = UserID.from_string(u) yield self.handlers.presence_handler.drop( observer_user=user, observed_user=dropped_user ) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index dc6eb424b0..39297930c8 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -16,7 +16,8 @@ """ This module contains REST servlets to do with profile: /profile/ """ from twisted.internet import defer -from base import RestServlet, client_path_pattern +from .base import RestServlet, client_path_pattern +from synapse.types import UserID import json @@ -26,7 +27,7 @@ class ProfileDisplaynameRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) displayname = yield self.handlers.profile_handler.get_displayname( user, @@ -37,7 +38,7 @@ class ProfileDisplaynameRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) try: content = json.loads(request.content.read()) @@ -59,7 +60,7 @@ class ProfileAvatarURLRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) avatar_url = yield self.handlers.profile_handler.get_avatar_url( user, @@ -70,7 +71,7 @@ class ProfileAvatarURLRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) try: content = json.loads(request.content.read()) @@ -92,7 +93,7 @@ class ProfileRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) displayname = yield self.handlers.profile_handler.get_displayname( user, diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 48bba2a5f3..c5837b3403 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -20,6 +20,7 @@ from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID import json import logging @@ -289,7 +290,7 @@ class RoomMemberListRestServlet(RestServlet): for event in members["chunk"]: # FIXME: should probably be state_key here, not user_id - target_user = self.hs.parse_userid(event["user_id"]) + target_user = UserID.from_string(event["user_id"]) # Presence is an optional cache; don't fail if we can't fetch it try: presence_handler = self.handlers.presence_handler @@ -478,7 +479,7 @@ class RoomTypingRestServlet(RestServlet): auth_user = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) - target_user = self.hs.parse_userid(urllib.unquote(user_id)) + target_user = UserID.from_string(urllib.unquote(user_id)) content = _parse_json(request) diff --git a/synapse/server.py b/synapse/server.py index 476d809374..52a21aaf78 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -127,12 +127,6 @@ class BaseHomeServer(object): # TODO: Why are these parse_ methods so high up along with other globals? # Surely these should be in a util package or in the api package? - # Other utility methods - def parse_userid(self, s): - """Parse the string given by 's' as a User ID and return a UserID - object.""" - return UserID.from_string(s) - def parse_roomalias(self, s): """Parse the string given by 's' as a Room Alias and return a RoomAlias object.""" diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index e59e65529b..c69dd995ce 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -20,6 +20,7 @@ from collections import namedtuple from ._base import SQLBaseStore from synapse.api.constants import Membership +from synapse.types import UserID import logging @@ -39,7 +40,7 @@ class RoomMemberStore(SQLBaseStore): """ try: target_user_id = event.state_key - domain = self.hs.parse_userid(target_user_id).domain + domain = UserID.from_string(target_user_id).domain except: logger.exception( "Failed to parse target_user_id=%s", target_user_id @@ -84,7 +85,7 @@ class RoomMemberStore(SQLBaseStore): for e in member_events: try: joined_domains.add( - self.hs.parse_userid(e.state_key).domain + UserID.from_string(e.state_key).domain ) except: # FIXME: How do we deal with invalid user ids in the db? diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 56e90177f1..5621a8afaf 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -17,7 +17,7 @@ from tests import unittest from twisted.internet import defer, reactor -from mock import Mock, call, ANY, NonCallableMock, patch +from mock import Mock, call, ANY, NonCallableMock import json from tests.utils import ( @@ -31,6 +31,7 @@ from synapse.api.errors import SynapseError from synapse.handlers.presence import PresenceHandler, UserPresenceCache from synapse.streams.config import SourcePaginationConfig from synapse.storage.transactions import DestinationsTable +from synapse.types import UserID OFFLINE = PresenceState.OFFLINE UNAVAILABLE = PresenceState.UNAVAILABLE @@ -170,9 +171,9 @@ class PresenceTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp_users(self, hs): # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") + self.u_clementine = UserID.from_string("@clementine:test") for u in self.u_apple, self.u_banana, self.u_clementine: yield self.datastore.create_presence(u.localpart) @@ -182,10 +183,10 @@ class PresenceTestCase(unittest.TestCase): ) # ID of a local user that does not exist - self.u_durian = hs.parse_userid("@durian:test") + self.u_durian = UserID.from_string("@durian:test") # A remote user - self.u_cabbage = hs.parse_userid("@cabbage:elsewhere") + self.u_cabbage = UserID.from_string("@cabbage:elsewhere") class MockedDatastorePresenceTestCase(PresenceTestCase): @@ -250,16 +251,16 @@ class MockedDatastorePresenceTestCase(PresenceTestCase): @defer.inlineCallbacks def setUp_users(self, hs): # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") - self.u_durian = hs.parse_userid("@durian:test") - self.u_elderberry = hs.parse_userid("@elderberry:test") - self.u_fig = hs.parse_userid("@fig:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") + self.u_clementine = UserID.from_string("@clementine:test") + self.u_durian = UserID.from_string("@durian:test") + self.u_elderberry = UserID.from_string("@elderberry:test") + self.u_fig = UserID.from_string("@fig:test") # Remote user - self.u_onion = hs.parse_userid("@onion:farm") - self.u_potato = hs.parse_userid("@potato:remote") + self.u_onion = UserID.from_string("@onion:farm") + self.u_potato = UserID.from_string("@potato:remote") yield diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 0584e4c8b9..3cdbb186ae 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -27,6 +27,7 @@ from synapse.server import HomeServer from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler from synapse.handlers.profile import ProfileHandler +from synapse.types import UserID OFFLINE = PresenceState.OFFLINE @@ -136,12 +137,12 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): lambda u: defer.succeed([])) # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") + self.u_clementine = UserID.from_string("@clementine:test") # Remote user - self.u_potato = hs.parse_userid("@potato:remote") + self.u_potato = UserID.from_string("@potato:remote") self.mock_get_joined = ( self.datastore.get_rooms_for_user_where_membership_is diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 25b172aa5e..7b9590c110 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -22,7 +22,7 @@ from mock import Mock, NonCallableMock from synapse.api.errors import AuthError from synapse.server import HomeServer from synapse.handlers.profile import ProfileHandler -from synapse.api.constants import Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -71,9 +71,9 @@ class ProfileTestCase(unittest.TestCase): self.store = hs.get_datastore() - self.frank = hs.parse_userid("@1234ABCD:test") - self.bob = hs.parse_userid("@4567:test") - self.alice = hs.parse_userid("@alice:remote") + self.frank = UserID.from_string("@1234ABCD:test") + self.bob = UserID.from_string("@4567:test") + self.alice = UserID.from_string("@alice:remote") yield self.store.create_profile(self.frank.localpart) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index d3253b48b8..9a23b3812d 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -15,12 +15,13 @@ from twisted.internet import defer -from tests import unittest +from .. import unittest from synapse.api.constants import EventTypes, Membership from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler from synapse.handlers.profile import ProfileHandler from synapse.server import HomeServer +from synapse.types import UserID from ..utils import MockKey from mock import Mock, NonCallableMock @@ -164,7 +165,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): event, context=context, ) self.notifier.on_new_room_event.assert_called_once_with( - event, extra_users=[self.hs.parse_userid(target_user_id)] + event, extra_users=[UserID.from_string(target_user_id)] ) self.assertFalse(self.datastore.get_room.called) self.assertFalse(self.datastore.store_room.called) @@ -174,7 +175,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def test_simple_join(self): room_id = "!foo:red" user_id = "@bob:red" - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) join_signal_observer = Mock() self.distributor.observe("user_joined_room", join_signal_observer) @@ -252,7 +253,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def test_simple_leave(self): room_id = "!foo:red" user_id = "@bob:red" - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) builder = self.hs.get_event_builder_factory().new({ "type": EventTypes.Member, diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 6a498b23a4..8a7fc028d1 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -27,6 +27,7 @@ from synapse.server import HomeServer from synapse.handlers.typing import TypingNotificationHandler from synapse.storage.transactions import DestinationsTable +from synapse.types import UserID def _expect_edu(destination, edu_type, content, origin="test"): @@ -153,11 +154,11 @@ class TypingNotificationsTestCase(unittest.TestCase): self.auth.check_joined_room = check_joined_room # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") # Remote user - self.u_onion = hs.parse_userid("@onion:farm") + self.u_onion = UserID.from_string("@onion:farm") @defer.inlineCallbacks def test_started_typing_local(self): diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 783720ac29..65d5cc4916 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -27,6 +27,7 @@ from synapse.handlers.presence import PresenceHandler from synapse.server import HomeServer from synapse.rest.client.v1 import presence from synapse.rest.client.v1 import events +from synapse.types import UserID OFFLINE = PresenceState.OFFLINE @@ -71,7 +72,7 @@ class PresenceStateTestCase(unittest.TestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(myid), + "user": UserID.from_string(myid), "admin": False, "device_id": None, } @@ -90,7 +91,7 @@ class PresenceStateTestCase(unittest.TestCase): presence.register_servlets(hs, self.mock_resource) - self.u_apple = hs.parse_userid(myid) + self.u_apple = UserID.from_string(myid) @defer.inlineCallbacks def test_get_my_status(self): @@ -161,12 +162,12 @@ class PresenceListTestCase(unittest.TestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(myid), + "user": UserID.from_string(myid), "admin": False, "device_id": None, } - room_member_handler = hs.handlers.room_member_handler = Mock( + hs.handlers.room_member_handler = Mock( spec=[ "get_rooms_for_user", ] @@ -176,8 +177,8 @@ class PresenceListTestCase(unittest.TestCase): presence.register_servlets(hs, self.mock_resource) - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") @defer.inlineCallbacks def test_get_my_list(self): @@ -281,7 +282,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.get_clock().time_msec.return_value = 1000000 def _get_user_by_req(req=None): - return hs.parse_userid(myid) + return UserID.from_string(myid) hs.get_auth().get_user_by_req = _get_user_by_req @@ -322,8 +323,8 @@ class PresenceEventStreamTestCase(unittest.TestCase): self.presence = hs.get_handlers().presence_handler - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") @defer.inlineCallbacks def test_shortpoll(self): diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 5b5c3edc22..39cd68d829 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -24,6 +24,7 @@ from ....utils import MockHttpResource, MockKey from synapse.api.errors import SynapseError, AuthError from synapse.server import HomeServer +from synapse.types import UserID from synapse.rest.client.v1 import profile @@ -57,7 +58,7 @@ class ProfileTestCase(unittest.TestCase): ) def _get_user_by_req(request=None): - return hs.parse_userid(myid) + return UserID.from_string(myid) hs.get_auth().get_user_by_req = _get_user_by_req diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 12f8040541..76ed550b75 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -22,13 +22,10 @@ import synapse.rest.client.v1.room from synapse.api.constants import Membership from synapse.server import HomeServer +from synapse.types import UserID -from tests import unittest - -# python imports import json import urllib -import types from ....utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase @@ -70,7 +67,7 @@ class RoomPermissionsTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -466,7 +463,7 @@ class RoomsMemberListTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -555,7 +552,7 @@ class RoomsCreateTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -657,7 +654,7 @@ class RoomTopicTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -773,7 +770,7 @@ class RoomMemberStateTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -909,7 +906,7 @@ class RoomMessagesTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -1013,7 +1010,7 @@ class RoomInitialSyncTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -1028,7 +1025,7 @@ class RoomInitialSyncTestCase(RestTestCase): # Since I'm getting my own presence I need to exist as far as presence # is concerned. hs.get_handlers().presence_handler.registered_user( - hs.parse_userid(self.user_id) + UserID.from_string(self.user_id) ) # create the room diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 647bcebfd8..c89b37d004 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -20,6 +20,7 @@ from twisted.internet import defer import synapse.rest.client.v1.room from synapse.server import HomeServer +from synapse.types import UserID from ....utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase @@ -69,7 +70,7 @@ class RoomTypingTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -82,7 +83,7 @@ class RoomTypingTestCase(RestTestCase): def get_room_members(room_id): if room_id == self.room_id: - return defer.succeed([hs.parse_userid(self.user_id)]) + return defer.succeed([UserID.from_string(self.user_id)]) else: return defer.succeed([]) diff --git a/tests/storage/test_presence.py b/tests/storage/test_presence.py index 9655d3cf42..1ab193736b 100644 --- a/tests/storage/test_presence.py +++ b/tests/storage/test_presence.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.presence import PresenceStore +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockClock @@ -37,8 +38,8 @@ class PresenceStoreTestCase(unittest.TestCase): self.store = PresenceStore(hs) - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") @defer.inlineCallbacks def test_state(self): diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index 5d36723c28..84381241bc 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.profile import ProfileStore +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool @@ -36,7 +37,7 @@ class ProfileStoreTestCase(unittest.TestCase): self.store = ProfileStore(hs) - self.u_frank = hs.parse_userid("@frank:test") + self.u_frank = UserID.from_string("@frank:test") @defer.inlineCallbacks def test_displayname(self): diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index 9806fbc69b..a16ccad881 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -48,8 +49,8 @@ class RedactionTestCase(unittest.TestCase): self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler - self.u_alice = hs.parse_userid("@alice:test") - self.u_bob = hs.parse_userid("@bob:test") + self.u_alice = UserID.from_string("@alice:test") + self.u_bob = UserID.from_string("@bob:test") self.room1 = hs.parse_roomid("!abc123:test") diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index e7739776ec..c6bfde069a 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool @@ -40,7 +41,7 @@ class RoomStoreTestCase(unittest.TestCase): self.room = hs.parse_roomid("!abcde:test") self.alias = hs.parse_roomalias("#a-room-name:test") - self.u_creator = hs.parse_userid("@creator:test") + self.u_creator = UserID.from_string("@creator:test") yield self.store.store_room(self.room.to_string(), room_creator_user_id=self.u_creator.to_string(), diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index a23a8189df..6b7930b1d8 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -49,11 +50,11 @@ class RoomMemberStoreTestCase(unittest.TestCase): self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler - self.u_alice = hs.parse_userid("@alice:test") - self.u_bob = hs.parse_userid("@bob:test") + self.u_alice = UserID.from_string("@alice:test") + self.u_bob = UserID.from_string("@bob:test") # User elsewhere on another host - self.u_charlie = hs.parse_userid("@charlie:elsewhere") + self.u_charlie = UserID.from_string("@charlie:elsewhere") self.room = hs.parse_roomid("!abc123:test") diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 9247fc579e..d7c7f64d5e 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -48,8 +49,8 @@ class StreamStoreTestCase(unittest.TestCase): self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler - self.u_alice = hs.parse_userid("@alice:test") - self.u_bob = hs.parse_userid("@bob:test") + self.u_alice = UserID.from_string("@alice:test") + self.u_bob = UserID.from_string("@bob:test") self.room1 = hs.parse_roomid("!abc123:test") self.room2 = hs.parse_roomid("!xyx987:test") diff --git a/tests/test_types.py b/tests/test_types.py index bfb9e6f548..2de7f22ab0 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -42,12 +42,6 @@ class UserIDTestCase(unittest.TestCase): self.assertTrue(userA == userAagain) self.assertTrue(userA != userB) - def test_via_homeserver(self): - user = mock_homeserver.parse_userid("@3456ijkl:my.domain") - - self.assertEquals("3456ijkl", user.localpart) - self.assertEquals("my.domain", user.domain) - class RoomAliasTestCase(unittest.TestCase): -- cgit 1.4.1 From 1c06c48ce2db3c6355e29de1533aebf36bc3775b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 11:55:12 +0000 Subject: Replace hs.parse_roomid with RoomID.from_string --- synapse/handlers/room.py | 2 -- synapse/rest/client/v1/room.py | 4 ++-- synapse/server.py | 7 +------ tests/storage/test_directory.py | 3 ++- tests/storage/test_redaction.py | 4 ++-- tests/storage/test_room.py | 6 +++--- tests/storage/test_roommember.py | 4 ++-- tests/storage/test_stream.py | 6 +++--- 8 files changed, 15 insertions(+), 21 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0242288c4e..edb96cec83 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -246,8 +246,6 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def get_room_members(self, room_id): - hs = self.hs - users = yield self.store.get_users_in_room(room_id) defer.returnValue([UserID.from_string(u) for u in users]) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c5837b3403..f0a9c932c1 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -20,7 +20,7 @@ from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID import json import logging @@ -227,7 +227,7 @@ class JoinRoomAliasServlet(RestServlet): identifier = self.hs.parse_roomalias(room_identifier) is_room_alias = True except SynapseError: - identifier = self.hs.parse_roomid(room_identifier) + identifier = RoomID.from_string(room_identifier) # TODO: Support for specifying the home server to join with? diff --git a/synapse/server.py b/synapse/server.py index 52a21aaf78..4dfff04277 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -26,7 +26,7 @@ from synapse.api.auth import Auth from synapse.handlers import Handlers from synapse.state import StateHandler from synapse.storage import DataStore -from synapse.types import UserID, RoomAlias, RoomID, EventID +from synapse.types import RoomAlias, EventID from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager @@ -132,11 +132,6 @@ class BaseHomeServer(object): object.""" return RoomAlias.from_string(s) - def parse_roomid(self, s): - """Parse the string given by 's' as a Room ID and return a RoomID - object.""" - return RoomID.from_string(s) - def parse_eventid(self, s): """Parse the string given by 's' as a Event ID and return a EventID object.""" diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py index e9c242cc07..1bc6391766 100644 --- a/tests/storage/test_directory.py +++ b/tests/storage/test_directory.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.directory import DirectoryStore +from synapse.types import RoomID from tests.utils import SQLiteMemoryDbPool @@ -37,7 +38,7 @@ class DirectoryStoreTestCase(unittest.TestCase): self.store = DirectoryStore(hs) - self.room = hs.parse_roomid("!abcde:test") + self.room = RoomID.from_string("!abcde:test") self.alias = hs.parse_roomalias("#my-room:test") @defer.inlineCallbacks diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index a16ccad881..0713dfab64 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -52,7 +52,7 @@ class RedactionTestCase(unittest.TestCase): self.u_alice = UserID.from_string("@alice:test") self.u_bob = UserID.from_string("@bob:test") - self.room1 = hs.parse_roomid("!abc123:test") + self.room1 = RoomID.from_string("!abc123:test") self.depth = 1 diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index c6bfde069a..baec3a3bb9 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool @@ -39,7 +39,7 @@ class RoomStoreTestCase(unittest.TestCase): # management of the 'room_aliases' table self.store = hs.get_datastore() - self.room = hs.parse_roomid("!abcde:test") + self.room = RoomID.from_string("!abcde:test") self.alias = hs.parse_roomalias("#a-room-name:test") self.u_creator = UserID.from_string("@creator:test") @@ -98,7 +98,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): self.store = hs.get_datastore() self.event_factory = hs.get_event_factory(); - self.room = hs.parse_roomid("!abcde:test") + self.room = RoomID.from_string("!abcde:test") yield self.store.store_room(self.room.to_string(), room_creator_user_id="@creator:text", diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 6b7930b1d8..2b9048e2a9 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -56,7 +56,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): # User elsewhere on another host self.u_charlie = UserID.from_string("@charlie:elsewhere") - self.room = hs.parse_roomid("!abc123:test") + self.room = RoomID.from_string("!abc123:test") @defer.inlineCallbacks def inject_room_member(self, room, user, membership, replaces_state=None): diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index d7c7f64d5e..b7f6e2aa80 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -52,8 +52,8 @@ class StreamStoreTestCase(unittest.TestCase): self.u_alice = UserID.from_string("@alice:test") self.u_bob = UserID.from_string("@bob:test") - self.room1 = hs.parse_roomid("!abc123:test") - self.room2 = hs.parse_roomid("!xyx987:test") + self.room1 = RoomID.from_string("!abc123:test") + self.room2 = RoomID.from_string("!xyx987:test") self.depth = 1 -- cgit 1.4.1 From ada711504efb4dd25fe1123d38a0b2d196b9890a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 13:21:58 +0000 Subject: Replace hs.parse_roomalias with RoomAlias.from_string --- synapse/handlers/directory.py | 3 ++- synapse/rest/client/v1/directory.py | 9 +++++---- synapse/rest/client/v1/room.py | 4 ++-- synapse/server.py | 7 +------ tests/handlers/test_directory.py | 7 ++++--- tests/storage/test_directory.py | 4 ++-- tests/storage/test_room.py | 4 ++-- tests/test_types.py | 6 ------ 8 files changed, 18 insertions(+), 26 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 91fceda2ac..58e9a91562 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -19,6 +19,7 @@ from ._base import BaseHandler from synapse.api.errors import SynapseError, Codes, CodeMessageException from synapse.api.constants import EventTypes +from synapse.types import RoomAlias import logging @@ -122,7 +123,7 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def on_directory_query(self, args): - room_alias = self.hs.parse_roomalias(args["room_alias"]) + room_alias = RoomAlias.from_string(args["room_alias"]) if not self.hs.is_mine(room_alias): raise SynapseError( 400, "Room Alias is not hosted on this Home Server" diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 7ff44fdd9e..1f33ec9e81 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -17,7 +17,8 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError, Codes -from base import RestServlet, client_path_pattern +from synapse.types import RoomAlias +from .base import RestServlet, client_path_pattern import json import logging @@ -35,7 +36,7 @@ class ClientDirectoryServer(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_alias): - room_alias = self.hs.parse_roomalias(room_alias) + room_alias = RoomAlias.from_string(room_alias) dir_handler = self.handlers.directory_handler res = yield dir_handler.get_association(room_alias) @@ -53,7 +54,7 @@ class ClientDirectoryServer(RestServlet): logger.debug("Got content: %s", content) - room_alias = self.hs.parse_roomalias(room_alias) + room_alias = RoomAlias.from_string(room_alias) logger.debug("Got room name: %s", room_alias.to_string()) @@ -92,7 +93,7 @@ class ClientDirectoryServer(RestServlet): dir_handler = self.handlers.directory_handler - room_alias = self.hs.parse_roomalias(room_alias) + room_alias = RoomAlias.from_string(room_alias) yield dir_handler.delete_association( user.to_string(), room_alias diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f0a9c932c1..42712d4a7c 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -20,7 +20,7 @@ from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID, RoomID +from synapse.types import UserID, RoomID, RoomAlias import json import logging @@ -224,7 +224,7 @@ class JoinRoomAliasServlet(RestServlet): identifier = None is_room_alias = False try: - identifier = self.hs.parse_roomalias(room_identifier) + identifier = RoomAlias.from_string(room_identifier) is_room_alias = True except SynapseError: identifier = RoomID.from_string(room_identifier) diff --git a/synapse/server.py b/synapse/server.py index 4dfff04277..41a26ad91a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -26,7 +26,7 @@ from synapse.api.auth import Auth from synapse.handlers import Handlers from synapse.state import StateHandler from synapse.storage import DataStore -from synapse.types import RoomAlias, EventID +from synapse.types import EventID from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager @@ -127,11 +127,6 @@ class BaseHomeServer(object): # TODO: Why are these parse_ methods so high up along with other globals? # Surely these should be in a util package or in the api package? - def parse_roomalias(self, s): - """Parse the string given by 's' as a Room Alias and return a RoomAlias - object.""" - return RoomAlias.from_string(s) - def parse_eventid(self, s): """Parse the string given by 's' as a Event ID and return a EventID object.""" diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 8e164e4be0..22119de46a 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -21,6 +21,7 @@ from mock import Mock from synapse.server import HomeServer from synapse.handlers.directory import DirectoryHandler +from synapse.types import RoomAlias from tests.utils import SQLiteMemoryDbPool, MockKey @@ -65,9 +66,9 @@ class DirectoryTestCase(unittest.TestCase): self.store = hs.get_datastore() - self.my_room = hs.parse_roomalias("#my-room:test") - self.your_room = hs.parse_roomalias("#your-room:test") - self.remote_room = hs.parse_roomalias("#another:remote") + self.my_room = RoomAlias.from_string("#my-room:test") + self.your_room = RoomAlias.from_string("#your-room:test") + self.remote_room = RoomAlias.from_string("#another:remote") @defer.inlineCallbacks def test_get_local_association(self): diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py index 1bc6391766..bc9ebf35e2 100644 --- a/tests/storage/test_directory.py +++ b/tests/storage/test_directory.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.directory import DirectoryStore -from synapse.types import RoomID +from synapse.types import RoomID, RoomAlias from tests.utils import SQLiteMemoryDbPool @@ -39,7 +39,7 @@ class DirectoryStoreTestCase(unittest.TestCase): self.store = DirectoryStore(hs) self.room = RoomID.from_string("!abcde:test") - self.alias = hs.parse_roomalias("#my-room:test") + self.alias = RoomAlias.from_string("#my-room:test") @defer.inlineCallbacks def test_room_to_alias(self): diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index baec3a3bb9..71e5d34143 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes -from synapse.types import UserID, RoomID +from synapse.types import UserID, RoomID, RoomAlias from tests.utils import SQLiteMemoryDbPool @@ -40,7 +40,7 @@ class RoomStoreTestCase(unittest.TestCase): self.store = hs.get_datastore() self.room = RoomID.from_string("!abcde:test") - self.alias = hs.parse_roomalias("#a-room-name:test") + self.alias = RoomAlias.from_string("#a-room-name:test") self.u_creator = UserID.from_string("@creator:test") yield self.store.store_room(self.room.to_string(), diff --git a/tests/test_types.py b/tests/test_types.py index 2de7f22ab0..b29a8415b1 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -56,9 +56,3 @@ class RoomAliasTestCase(unittest.TestCase): room = RoomAlias("channel", "my.domain") self.assertEquals(room.to_string(), "#channel:my.domain") - - def test_via_homeserver(self): - room = mock_homeserver.parse_roomalias("#elsewhere:my.domain") - - self.assertEquals("elsewhere", room.localpart) - self.assertEquals("my.domain", room.domain) -- cgit 1.4.1 From 6188c4f69c2f902410b43bc50c0ae8d488b4d93c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:23:10 +0000 Subject: make per-device rules work --- synapse/rest/client/v1/push_rule.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 6f108431b2..417fd368d7 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -73,7 +73,7 @@ class PushRuleRestServlet(RestServlet): spec['device'] = device return spec - def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj): + def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj, device=None): if rule_template in ['override', 'underride']: if 'conditions' not in req_obj: raise InvalidRuleException("Missing 'conditions'") @@ -104,6 +104,12 @@ class PushRuleRestServlet(RestServlet): else: raise InvalidRuleException("Unknown rule template: %s" % (rule_template)) + if device: + conditions.append({ + 'kind': 'device', + 'instance_handle': device + }) + if 'actions' not in req_obj: raise InvalidRuleException("No actions found") actions = req_obj['actions'] @@ -144,7 +150,8 @@ class PushRuleRestServlet(RestServlet): (conditions, actions) = self.rule_tuple_from_request_object( spec['template'], spec['rule_id'], - content + content, + device=spec['device'] if 'device' in spec else None ) except InvalidRuleException as e: raise SynapseError(400, e.message) @@ -200,11 +207,11 @@ class PushRuleRestServlet(RestServlet): if not instance_handle: continue if instance_handle not in rules['device']: - rules['device'][instance_handle] = [] + rules['device'][instance_handle] = {} rules['device'][instance_handle] = \ _add_empty_priority_class_arrays(rules['device'][instance_handle]) - rulearray = rules['device'][instance_handle] + rulearray = rules['device'][instance_handle][template_name] else: rulearray = rules['global'][template_name] @@ -227,7 +234,10 @@ class PushRuleRestServlet(RestServlet): elif path[0] == 'device': path = path[1:] if path == []: - raise UnrecognizedRequestError + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + if path[0] == '': + defer.returnValue((200, rules['device'])) + instance_handle = path[0] if instance_handle not in rules['device']: ret = {} -- cgit 1.4.1 From c4652d7772a0d9b374fc178502a71efd03d35d48 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 13:25:07 +0000 Subject: Remove hs.parse_eventid --- synapse/server.py | 9 --------- 1 file changed, 9 deletions(-) (limited to 'synapse') diff --git a/synapse/server.py b/synapse/server.py index 41a26ad91a..32013b1a91 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -26,7 +26,6 @@ from synapse.api.auth import Auth from synapse.handlers import Handlers from synapse.state import StateHandler from synapse.storage import DataStore -from synapse.types import EventID from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager @@ -124,14 +123,6 @@ class BaseHomeServer(object): setattr(BaseHomeServer, "get_%s" % (depname), _get) - # TODO: Why are these parse_ methods so high up along with other globals? - # Surely these should be in a util package or in the api package? - - def parse_eventid(self, s): - """Parse the string given by 's' as a Event ID and return a EventID - object.""" - return EventID.from_string(s) - def serialize_event(self, e, as_client_event=True): return serialize_event(self, e, as_client_event) -- cgit 1.4.1 From 54c689c8199336b819c632a3e996120cb13007db Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:25:14 +0000 Subject: stray space --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 417fd368d7..9cb2494035 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -65,7 +65,7 @@ class PushRuleRestServlet(RestServlet): rule_id = path[0] spec = { - 'scope' : scope, + 'scope': scope, 'template': template, 'rule_id': rule_id } -- cgit 1.4.1 From 98e1080555965c650e09ed09bdac3b52daeda123 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:25:36 +0000 Subject: redundant parens --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 9cb2494035..46b8c3f625 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -122,7 +122,7 @@ class PushRuleRestServlet(RestServlet): else: raise InvalidRuleException("Unrecognised action") - return (conditions, actions) + return conditions, actions def priority_class_from_spec(self, spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): -- cgit 1.4.1 From d3e72b4d8788e64cd4b1d9668382639476730b4d Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:25:58 +0000 Subject: Make string format tuple an actual tuple --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 46b8c3f625..35ffcba3df 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -102,7 +102,7 @@ class PushRuleRestServlet(RestServlet): 'pattern': req_obj['pattern'] }] else: - raise InvalidRuleException("Unknown rule template: %s" % (rule_template)) + raise InvalidRuleException("Unknown rule template: %s" % (rule_template,)) if device: conditions.append({ -- cgit 1.4.1 From b3f66ea6fb5871d7eeb71b466740adb61a89c0d2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:28:00 +0000 Subject: more pep8 --- synapse/rest/client/v1/push_rule.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 35ffcba3df..ce2f0febf4 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -249,7 +249,6 @@ class PushRuleRestServlet(RestServlet): else: raise UnrecognizedRequestError() - def on_OPTIONS(self, _): return 200, {} @@ -259,6 +258,7 @@ def _add_empty_priority_class_arrays(d): d[pc] = [] return d + def _instance_handle_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the @@ -289,6 +289,7 @@ def _filter_ruleset_with_path(ruleset, path): return r raise NotFoundError + def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device @@ -297,12 +298,13 @@ def _priority_class_to_template_name(pc): else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: return {k:rule[k] for k in ["rule_id", "conditions", "actions"]} elif template_name in ["sender", "room"]: - return {k:rule[k] for k in ["rule_id", "actions"]} + return {k: rule[k] for k in ["rule_id", "actions"]} elif template_name == 'content': if len(rule["conditions"]) != 1: return None -- cgit 1.4.1 From fc7a05c443c169b07b917bb2692bdd7c738824fb Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:36:01 +0000 Subject: more pep8 suggestions --- synapse/push/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 3ee652f3bc..47da31e500 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -74,7 +74,6 @@ class Pusher(object): defer.returnValue(ctx) - @defer.inlineCallbacks def start(self): if not self.last_token: @@ -116,7 +115,7 @@ class Pusher(object): processed = False if self._should_notify_for_event(single_event): rejected = yield self.dispatch_push(single_event) - if not rejected == False: + if not rejected is False: processed = True for pk in rejected: if pk != self.pushkey: -- cgit 1.4.1 From 4be637cb120f00e5de99c12a79e908e2f26da8af Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 14:09:51 +0000 Subject: Extract the client v1 base RestServlet to a separate class --- synapse/rest/client/v1/admin.py | 4 ++-- synapse/rest/client/v1/base.py | 40 +++++----------------------------- synapse/rest/client/v1/directory.py | 4 ++-- synapse/rest/client/v1/events.py | 6 ++--- synapse/rest/client/v1/initial_sync.py | 4 ++-- synapse/rest/client/v1/login.py | 8 +++---- synapse/rest/client/v1/presence.py | 6 ++--- synapse/rest/client/v1/profile.py | 8 +++---- synapse/rest/client/v1/register.py | 4 ++-- synapse/rest/client/v1/room.py | 28 ++++++++++++------------ synapse/rest/client/v1/voip.py | 4 ++-- 11 files changed, 44 insertions(+), 72 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 4aefb94053..1051d96f96 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -18,14 +18,14 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError from synapse.types import UserID -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern import logging logger = logging.getLogger(__name__) -class WhoisRestServlet(RestServlet): +class WhoisRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/admin/whois/(?P[^/]*)") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/base.py b/synapse/rest/client/v1/base.py index d005206b77..72332bdb10 100644 --- a/synapse/rest/client/v1/base.py +++ b/synapse/rest/client/v1/base.py @@ -13,7 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" This module contains base REST classes for constructing REST servlets. """ +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.http.servlet import RestServlet from synapse.api.urls import CLIENT_PREFIX from .transactions import HttpTransactionStore import re @@ -37,44 +40,13 @@ def client_path_pattern(path_regex): return re.compile("^" + CLIENT_PREFIX + path_regex) -class RestServlet(object): - - """ A Synapse REST Servlet. - - An implementing class can either provide its own custom 'register' method, - or use the automatic pattern handling provided by the base class. - - To use this latter, the implementing class instead provides a `PATTERN` - class attribute containing a pre-compiled regular expression. The automatic - register method will then use this method to register any of the following - instance methods associated with the corresponding HTTP method: - - on_GET - on_PUT - on_POST - on_DELETE - on_OPTIONS - - Automatically handles turning CodeMessageExceptions thrown by these methods - into the appropriate HTTP response. +class ClientV1RestServlet(RestServlet): + """A base Synapse REST Servlet for the client version 1 API. """ def __init__(self, hs): self.hs = hs - self.handlers = hs.get_handlers() self.builder_factory = hs.get_event_builder_factory() self.auth = hs.get_auth() self.txns = HttpTransactionStore() - - def register(self, http_server): - """ Register this servlet with the given HTTP server. """ - if hasattr(self, "PATTERN"): - pattern = self.PATTERN - - for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"): - if hasattr(self, "on_%s" % (method)): - method_handler = getattr(self, "on_%s" % (method)) - http_server.register_path(method, pattern, method_handler) - else: - raise NotImplementedError("RestServlet must register something.") diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 1f33ec9e81..15ae8749b8 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError, Codes from synapse.types import RoomAlias -from .base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern import json import logging @@ -31,7 +31,7 @@ def register_servlets(hs, http_server): ClientDirectoryServer(hs).register(http_server) -class ClientDirectoryServer(RestServlet): +class ClientDirectoryServer(ClientV1RestServlet): PATTERN = client_path_pattern("/directory/room/(?P[^/]*)$") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index c2515528ac..c69de56863 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig -from .base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern import logging @@ -26,7 +26,7 @@ import logging logger = logging.getLogger(__name__) -class EventStreamRestServlet(RestServlet): +class EventStreamRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/events$") DEFAULT_LONGPOLL_TIME_MS = 30000 @@ -61,7 +61,7 @@ class EventStreamRestServlet(RestServlet): # TODO: Unit test gets, with and without auth, with different kinds of events. -class EventRestServlet(RestServlet): +class EventRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/events/(?P[^/]*)$") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index b13d56b286..357fa845b4 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -16,11 +16,11 @@ from twisted.internet import defer from synapse.streams.config import PaginationConfig -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern # TODO: Needs unit testing -class InitialSyncRestServlet(RestServlet): +class InitialSyncRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/initialSync$") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 6b8deff67b..7116ac98e8 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -17,12 +17,12 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.types import UserID -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern import json -class LoginRestServlet(RestServlet): +class LoginRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login$") PASS_TYPE = "m.login.password" @@ -64,7 +64,7 @@ class LoginRestServlet(RestServlet): defer.returnValue((200, result)) -class LoginFallbackRestServlet(RestServlet): +class LoginFallbackRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login/fallback$") def on_GET(self, request): @@ -73,7 +73,7 @@ class LoginFallbackRestServlet(RestServlet): return (200, {}) -class PasswordResetRestServlet(RestServlet): +class PasswordResetRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login/reset") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 22fcb7d7d0..b6c207e662 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.types import UserID -from .base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern import json import logging @@ -27,7 +27,7 @@ import logging logger = logging.getLogger(__name__) -class PresenceStatusRestServlet(RestServlet): +class PresenceStatusRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/presence/(?P[^/]*)/status") @defer.inlineCallbacks @@ -72,7 +72,7 @@ class PresenceStatusRestServlet(RestServlet): return (200, {}) -class PresenceListRestServlet(RestServlet): +class PresenceListRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/presence/list/(?P[^/]*)") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 39297930c8..24f8d56952 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -16,13 +16,13 @@ """ This module contains REST servlets to do with profile: /profile/ """ from twisted.internet import defer -from .base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID import json -class ProfileDisplaynameRestServlet(RestServlet): +class ProfileDisplaynameRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") @defer.inlineCallbacks @@ -55,7 +55,7 @@ class ProfileDisplaynameRestServlet(RestServlet): return (200, {}) -class ProfileAvatarURLRestServlet(RestServlet): +class ProfileAvatarURLRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)/avatar_url") @defer.inlineCallbacks @@ -88,7 +88,7 @@ class ProfileAvatarURLRestServlet(RestServlet): return (200, {}) -class ProfileRestServlet(RestServlet): +class ProfileRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index e3b26902d9..c0423c2d45 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes from synapse.api.constants import LoginType -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor @@ -42,7 +42,7 @@ else: compare_digest = lambda a, b: a == b -class RegisterRestServlet(RestServlet): +class RegisterRestServlet(ClientV1RestServlet): """Handles registration with the home server. This servlet is in control of the registration flow; the registration diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 42712d4a7c..f06e3ddb98 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -16,7 +16,7 @@ """ This module contains REST servlets to do with rooms: /rooms/ """ from twisted.internet import defer -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership @@ -30,7 +30,7 @@ import urllib logger = logging.getLogger(__name__) -class RoomCreateRestServlet(RestServlet): +class RoomCreateRestServlet(ClientV1RestServlet): # No PATTERN; we have custom dispatch rules here def register(self, http_server): @@ -94,7 +94,7 @@ class RoomCreateRestServlet(RestServlet): # TODO: Needs unit testing for generic events -class RoomStateEventRestServlet(RestServlet): +class RoomStateEventRestServlet(ClientV1RestServlet): def register(self, http_server): # /room/$roomid/state/$eventtype no_state_key = "/rooms/(?P[^/]*)/state/(?P[^/]*)$" @@ -163,7 +163,7 @@ class RoomStateEventRestServlet(RestServlet): # TODO: Needs unit testing for generic events + feedback -class RoomSendEventRestServlet(RestServlet): +class RoomSendEventRestServlet(ClientV1RestServlet): def register(self, http_server): # /rooms/$roomid/send/$event_type[/$txn_id] @@ -206,7 +206,7 @@ class RoomSendEventRestServlet(RestServlet): # TODO: Needs unit testing for room ID + alias joins -class JoinRoomAliasServlet(RestServlet): +class JoinRoomAliasServlet(ClientV1RestServlet): def register(self, http_server): # /join/$room_identifier[/$txn_id] @@ -265,7 +265,7 @@ class JoinRoomAliasServlet(RestServlet): # TODO: Needs unit testing -class PublicRoomListRestServlet(RestServlet): +class PublicRoomListRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/publicRooms$") @defer.inlineCallbacks @@ -276,7 +276,7 @@ class PublicRoomListRestServlet(RestServlet): # TODO: Needs unit testing -class RoomMemberListRestServlet(RestServlet): +class RoomMemberListRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/members$") @defer.inlineCallbacks @@ -305,7 +305,7 @@ class RoomMemberListRestServlet(RestServlet): # TODO: Needs unit testing -class RoomMessageListRestServlet(RestServlet): +class RoomMessageListRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/messages$") @defer.inlineCallbacks @@ -329,7 +329,7 @@ class RoomMessageListRestServlet(RestServlet): # TODO: Needs unit testing -class RoomStateRestServlet(RestServlet): +class RoomStateRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/state$") @defer.inlineCallbacks @@ -345,7 +345,7 @@ class RoomStateRestServlet(RestServlet): # TODO: Needs unit testing -class RoomInitialSyncRestServlet(RestServlet): +class RoomInitialSyncRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/initialSync$") @defer.inlineCallbacks @@ -360,7 +360,7 @@ class RoomInitialSyncRestServlet(RestServlet): defer.returnValue((200, content)) -class RoomTriggerBackfill(RestServlet): +class RoomTriggerBackfill(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/backfill$") @defer.inlineCallbacks @@ -379,7 +379,7 @@ class RoomTriggerBackfill(RestServlet): # TODO: Needs unit testing -class RoomMembershipRestServlet(RestServlet): +class RoomMembershipRestServlet(ClientV1RestServlet): def register(self, http_server): # /rooms/$roomid/[invite|join|leave] @@ -431,7 +431,7 @@ class RoomMembershipRestServlet(RestServlet): defer.returnValue(response) -class RoomRedactEventRestServlet(RestServlet): +class RoomRedactEventRestServlet(ClientV1RestServlet): def register(self, http_server): PATTERN = ("/rooms/(?P[^/]*)/redact/(?P[^/]*)") register_txn_path(self, PATTERN, http_server) @@ -469,7 +469,7 @@ class RoomRedactEventRestServlet(RestServlet): defer.returnValue(response) -class RoomTypingRestServlet(RestServlet): +class RoomTypingRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern( "/rooms/(?P[^/]*)/typing/(?P[^/]*)$" ) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 011c35e69b..822d863ce6 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern import hmac @@ -23,7 +23,7 @@ import hashlib import base64 -class VoipRestServlet(RestServlet): +class VoipRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/voip/turnServer$") @defer.inlineCallbacks -- cgit 1.4.1 From e0bf18addf643bd72a0cd5cb8dbc127dc7a7e8d2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 14:16:28 +0000 Subject: Add RestServlet base class in synapse/http/servlet.py --- synapse/http/servlet.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 synapse/http/servlet.py (limited to 'synapse') diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py new file mode 100644 index 0000000000..d5ccf2742f --- /dev/null +++ b/synapse/http/servlet.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains base REST classes for constructing REST servlets. """ + +import logging + + +logger = logging.getLogger(__name__) + + +class RestServlet(object): + + """ A Synapse REST Servlet. + + An implementing class can either provide its own custom 'register' method, + or use the automatic pattern handling provided by the base class. + + To use this latter, the implementing class instead provides a `PATTERN` + class attribute containing a pre-compiled regular expression. The automatic + register method will then use this method to register any of the following + instance methods associated with the corresponding HTTP method: + + on_GET + on_PUT + on_POST + on_DELETE + on_OPTIONS + + Automatically handles turning CodeMessageExceptions thrown by these methods + into the appropriate HTTP response. + """ + + def register(self, http_server): + """ Register this servlet with the given HTTP server. """ + if hasattr(self, "PATTERN"): + pattern = self.PATTERN + + for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"): + if hasattr(self, "on_%s" % (method)): + method_handler = getattr(self, "on_%s" % (method)) + http_server.register_path(method, pattern, method_handler) + else: + raise NotImplementedError("RestServlet must register something.") -- cgit 1.4.1 From 3b9cc882a50c886afd7d2cf1eaa7e02e8b0d0d51 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 23 Jan 2015 15:42:52 +0000 Subject: Add storage method have_events --- synapse/storage/__init__.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 015fcc8775..4f09909607 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -422,6 +422,35 @@ class DataStore(RoomMemberStore, RoomStore, ], ) + def have_events(self, event_ids): + """Given a list of event ids, check if we have already processed them. + + Returns: + dict: Has an entry for each event id we already have seen. Maps to + the rejected reason string if we rejected the event, else maps to + None. + """ + def f(txn): + sql = ( + "SELECT e.event_id, reason FROM events as e " + "LEFT JOIN rejections as r ON e.event_id = r.event_id " + "WHERE event_id = ?" + ) + + res = {} + for event_id in event_ids: + txn.execute(sql, (event_id,)) + row = txn.fetchone() + if row: + _, rejected = row + res[event_id] = rejected + + return res + + return self.runInteraction( + "have_events", f, + ) + def schema_path(schema): """ Get a filesystem path for the named database schema -- cgit 1.4.1 From f21f9fa3c51db49212c42adfe6972025e1d27a15 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 17:07:06 +0000 Subject: Use push settings! --- synapse/push/__init__.py | 91 +++++++++++++++++++++++++++++++++---- synapse/push/httppusher.py | 9 ++-- synapse/rest/client/v1/push_rule.py | 43 ++++++++++++------ 3 files changed, 117 insertions(+), 26 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 47da31e500..53d3319699 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -21,6 +21,8 @@ from synapse.types import StreamToken import synapse.util.async import logging +import fnmatch +import json logger = logging.getLogger(__name__) @@ -29,6 +31,7 @@ class Pusher(object): INITIAL_BACKOFF = 1000 MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 + DEFAULT_ACTIONS = ['notify'] def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, @@ -37,7 +40,7 @@ class Pusher(object): self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() - self.instance_handle = instance_handle, + self.instance_handle = instance_handle self.user_name = user_name self.app_id = app_id self.app_display_name = app_display_name @@ -51,7 +54,8 @@ class Pusher(object): self.failing_since = failing_since self.alive = True - def _should_notify_for_event(self, ev): + @defer.inlineCallbacks + def _actions_for_event(self, ev): """ This should take into account notification settings that the user has configured both globally and per-room when we have the ability @@ -59,8 +63,47 @@ class Pusher(object): """ if ev['user_id'] == self.user_name: # let's assume you probably know about messages you sent yourself + defer.returnValue(['dont_notify']) + + rules = yield self.store.get_push_rules_for_user_name(self.user_name) + + for r in rules: + matches = True + + conditions = json.loads(r['conditions']) + actions = json.loads(r['actions']) + + for c in conditions: + matches &= self._event_fulfills_condition(ev, c) + # ignore rules with no actions (we have an explict 'dont_notify' + if len(actions) == 0: + logger.warn( + "Ignoring rule id %s with no actions for user %s" % + (r['rule_id'], r['user_name']) + ) + continue + if matches: + defer.returnValue(actions) + + defer.returnValue(Pusher.DEFAULT_ACTIONS) + + def _event_fulfills_condition(self, ev, condition): + if condition['kind'] == 'event_match': + if 'pattern' not in condition: + logger.warn("event_match condition with no pattern") + return False + pat = condition['pattern'] + + val = _value_for_dotted_key(condition['key'], ev) + if fnmatch.fnmatch(val, pat): + return True return False - return True + elif condition['kind'] == 'device': + if 'instance_handle' not in condition: + return True + return condition['instance_handle'] == self.instance_handle + else: + return True @defer.inlineCallbacks def get_context_for_event(self, ev): @@ -113,8 +156,23 @@ class Pusher(object): continue processed = False - if self._should_notify_for_event(single_event): - rejected = yield self.dispatch_push(single_event) + actions = yield self._actions_for_event(single_event) + tweaks = _tweaks_for_actions(actions) + + if len(actions) == 0: + logger.warn("Empty actions! Using default action.") + actions = Pusher.DEFAULT_ACTIONS + if 'notify' not in actions and 'dont_notify' not in actions: + logger.warn("Neither notify nor dont_notify in actions: adding default") + actions.extend(Pusher.DEFAULT_ACTIONS) + if 'dont_notify' in actions: + logger.debug( + "%s for %s: dont_notify", + single_event['event_id'], self.user_name + ) + processed = True + else: + rejected = yield self.dispatch_push(single_event, tweaks) if not rejected is False: processed = True for pk in rejected: @@ -133,8 +191,6 @@ class Pusher(object): yield self.hs.get_pusherpool().remove_pusher( self.app_id, pk ) - else: - processed = True if not self.alive: continue @@ -202,7 +258,7 @@ class Pusher(object): def stop(self): self.alive = False - def dispatch_push(self, p): + def dispatch_push(self, p, tweaks): """ Overridden by implementing classes to actually deliver the notification :param p: The event to notify for as a single event from the event stream @@ -214,6 +270,25 @@ class Pusher(object): pass +def _value_for_dotted_key(dotted_key, event): + parts = dotted_key.split(".") + val = event + while len(parts) > 0: + if parts[0] not in val: + return None + val = val[parts[0]] + parts = parts[1:] + return val + +def _tweaks_for_actions(actions): + tweaks = {} + for a in actions: + if not isinstance(a, dict): + continue + if 'set_sound' in a: + tweaks['sound'] = a['set_sound'] + return tweaks + class PusherConfigException(Exception): def __init__(self, msg): super(PusherConfigException, self).__init__(msg) \ No newline at end of file diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 46433ad4a9..25db1dded5 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -52,7 +52,7 @@ class HttpPusher(Pusher): del self.data_minus_url['url'] @defer.inlineCallbacks - def _build_notification_dict(self, event): + def _build_notification_dict(self, event, tweaks): # we probably do not want to push for every presence update # (we may want to be able to set up notifications when specific # people sign in, but we'd want to only deliver the pertinent ones) @@ -83,7 +83,8 @@ class HttpPusher(Pusher): 'app_id': self.app_id, 'pushkey': self.pushkey, 'pushkey_ts': long(self.pushkey_ts / 1000), - 'data': self.data_minus_url + 'data': self.data_minus_url, + 'tweaks': tweaks } ] } @@ -97,8 +98,8 @@ class HttpPusher(Pusher): defer.returnValue(d) @defer.inlineCallbacks - def dispatch_push(self, event): - notification_dict = yield self._build_notification_dict(event) + def dispatch_push(self, event, tweaks): + notification_dict = yield self._build_notification_dict(event, tweaks) if not notification_dict: defer.returnValue([]) try: diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index ce2f0febf4..9dc2c0e11e 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -96,10 +96,15 @@ class PushRuleRestServlet(RestServlet): elif rule_template == 'content': if 'pattern' not in req_obj: raise InvalidRuleException("Content rule missing 'pattern'") + pat = req_obj['pattern'] + if pat.strip("*?[]") == pat: + # no special glob characters so we assume the user means + # 'contains this string' rather than 'is this string' + pat = "*%s*" % (pat) conditions = [{ 'kind': 'event_match', 'key': 'content.body', - 'pattern': req_obj['pattern'] + 'pattern': pat }] else: raise InvalidRuleException("Unknown rule template: %s" % (rule_template,)) @@ -115,7 +120,7 @@ class PushRuleRestServlet(RestServlet): actions = req_obj['actions'] for a in actions: - if a in ['notify', 'dont-notify', 'coalesce']: + if a in ['notify', 'dont_notify', 'coalesce']: pass elif isinstance(a, dict) and 'set_sound' in a: pass @@ -124,21 +129,11 @@ class PushRuleRestServlet(RestServlet): return conditions, actions - def priority_class_from_spec(self, spec): - if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): - raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) - pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] - - if spec['scope'] == 'device': - pc += 5 - - return pc - @defer.inlineCallbacks def on_PUT(self, request): spec = self.rule_spec_from_path(request.postpath) try: - priority_class = self.priority_class_from_spec(spec) + priority_class = _priority_class_from_spec(spec) except InvalidRuleException as e: raise SynapseError(400, e.message) @@ -204,6 +199,7 @@ class PushRuleRestServlet(RestServlet): if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device rule instance_handle = _instance_handle_from_conditions(r["conditions"]) + r = _strip_device_condition(r) if not instance_handle: continue if instance_handle not in rules['device']: @@ -239,6 +235,7 @@ class PushRuleRestServlet(RestServlet): defer.returnValue((200, rules['device'])) instance_handle = path[0] + path = path[1:] if instance_handle not in rules['device']: ret = {} ret = _add_empty_priority_class_arrays(ret) @@ -290,10 +287,21 @@ def _filter_ruleset_with_path(ruleset, path): raise NotFoundError +def _priority_class_from_spec(spec): + if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): + raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) + pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] + + if spec['scope'] == 'device': + pc += len(PushRuleRestServlet.PRIORITY_CLASS_MAP) + + return pc + + def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device - prio_class_index = pc - PushRuleRestServlet.PRIORITY_CLASS_MAP['override'] + prio_class_index = pc - len(PushRuleRestServlet.PRIORITY_CLASS_MAP) return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[prio_class_index] else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] @@ -316,6 +324,13 @@ def _rule_to_template(rule): return ret +def _strip_device_condition(rule): + for i,c in enumerate(rule['conditions']): + if c['kind'] == 'device': + del rule['conditions'][i] + return rule + + class InvalidRuleException(Exception): pass -- cgit 1.4.1 From 5f84ba8ea1991dff279f0135f474d9debfd1419a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 17:49:37 +0000 Subject: Add API to delete push rules. --- synapse/rest/client/v1/push_rule.py | 41 ++++++++++++++++++++++++++++++++++++- synapse/storage/push_rule.py | 9 ++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 9dc2c0e11e..50bf5b9008 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -15,7 +15,8 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError +from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError, \ + StoreError from base import RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException @@ -175,6 +176,44 @@ class PushRuleRestServlet(RestServlet): defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_DELETE(self, request): + spec = self.rule_spec_from_path(request.postpath) + try: + priority_class = _priority_class_from_spec(spec) + except InvalidRuleException as e: + raise SynapseError(400, e.message) + + user = yield self.auth.get_user_by_req(request) + + if 'device' in spec: + rules = yield self.hs.get_datastore().get_push_rules_for_user_name( + user.to_string() + ) + + for r in rules: + conditions = json.loads(r['conditions']) + ih = _instance_handle_from_conditions(conditions) + if ih == spec['device'] and r['priority_class'] == priority_class: + yield self.hs.get_datastore().delete_push_rule( + user.to_string(), spec['rule_id'] + ) + defer.returnValue((200, {})) + raise NotFoundError() + else: + try: + yield self.hs.get_datastore().delete_push_rule( + user.to_string(), spec['rule_id'], + priority_class=priority_class + ) + defer.returnValue((200, {})) + except StoreError as e: + if e.code == 404: + raise NotFoundError() + else: + raise + + @defer.inlineCallbacks def on_GET(self, request): user = yield self.auth.get_user_by_req(request) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 2366090e09..ca04f2ccee 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -174,6 +174,15 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) + @defer.inlineCallbacks + def delete_push_rule(self, user_name, rule_id): + yield self._simple_delete_one( + PushRuleTable.table_name, + { + 'user_name': user_name, + 'rule_id': rule_id + } + ) class RuleNotFoundException(Exception): pass -- cgit 1.4.1 From 85419e12571ff07f2978d6013e592ac9bc207ee0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 23 Jan 2015 18:37:13 +0000 Subject: Stop complaining about Synapse Angular SDK 0.6.1 --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index b1fae991e0..4182ad990f 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], - "matrix_angular_sdk==0.6.0": ["syweb==0.6.0"], + "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], "Twisted>=14.0.0": ["twisted>=14.0.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], -- cgit 1.4.1 From e26340cee7049b6c36f4c3451ec7524fa6b80d1c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 18:31:29 +0000 Subject: Start implementing the v2_alpha sync API --- synapse/api/urls.py | 1 + synapse/http/servlet.py | 57 ++++++++++++ synapse/rest/client/v2_alpha/__init__.py | 33 +++++++ synapse/rest/client/v2_alpha/_base.py | 38 ++++++++ synapse/rest/client/v2_alpha/sync.py | 143 +++++++++++++++++++++++++++++++ 5 files changed, 272 insertions(+) create mode 100644 synapse/rest/client/v2_alpha/__init__.py create mode 100644 synapse/rest/client/v2_alpha/_base.py create mode 100644 synapse/rest/client/v2_alpha/sync.py (limited to 'synapse') diff --git a/synapse/api/urls.py b/synapse/api/urls.py index a299392049..693c0efda6 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -16,6 +16,7 @@ """Contains the URL paths to prefix various aspects of the server with. """ CLIENT_PREFIX = "/_matrix/client/api/v1" +CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha" FEDERATION_PREFIX = "/_matrix/federation/v1" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index d5ccf2742f..a4eb6c817c 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -15,6 +15,8 @@ """ This module contains base REST classes for constructing REST servlets. """ +from synapse.api.errors import SynapseError + import logging @@ -54,3 +56,58 @@ class RestServlet(object): http_server.register_path(method, pattern, method_handler) else: raise NotImplementedError("RestServlet must register something.") + + @staticmethod + def parse_integer(request, name, default=None, required=False): + if name in request.args: + try: + return int(request.args[name][0]) + except: + message = "Query parameter %r must be an integer" % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing integer query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_boolean(request, name, default=None, required=False): + if name in request.args: + try: + return { + "true": True, + "false": False, + }[request.args[name][0]] + except: + message = ( + "Boolean query parameter %r must be one of" + " ['true', 'false']" + ) % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing boolean query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_string(request, name, default=None, required=False, + allowed_values=None, param_type="string"): + if name in request.args: + value = request.args[name][0] + if allowed_values is not None and value not in allowed_values: + message = "Query parameter %r must be one of [%s]" % ( + name, ", ".join(repr(v) for v in allowed_values) + ) + raise SynapseError(message) + else: + return value + else: + if required: + message = "Missing %s query parameter %r" % (param_type, name) + raise SynapseError(400, message) + else: + return default diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py new file mode 100644 index 0000000000..e75f9d250f --- /dev/null +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from . import ( + sync, +) + +from synapse.http.server import JsonResource + + +class ClientV2AlphaRestResource(JsonResource): + """A resource for version 2 alpha of the matrix client API.""" + + def __init__(self, hs): + JsonResource.__init__(self) + self.register_servlets(self, hs) + + @staticmethod + def register_servlets(client_resource, hs): + sync.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py new file mode 100644 index 0000000000..22dc5cb862 --- /dev/null +++ b/synapse/rest/client/v2_alpha/_base.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def client_v2_pattern(path_regex): + """Creates a regex compiled client path with the correct client path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py new file mode 100644 index 0000000000..39bb5ec8e9 --- /dev/null +++ b/synapse/rest/client/v2_alpha/sync.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.http.servlet import RestServlet +from ._base import client_v2_pattern + +import logging + +logger = logging.getLogger(__name__) + + +class SyncRestServlet(RestServlet): + """ + + GET parameters:: + timeout(int): How long to wait for new events in milliseconds. + limit(int): Maxiumum number of events per room to return. + gap(bool): Create gaps the message history if limit is exceeded to + ensure that the client has the most recent messages. Defaults to + "true". + sort(str,str): tuple of sort key (e.g. "timeline") and direction + (e.g. "asc", "desc"). Defaults to "timeline,asc". + since(batch_token): Batch token when asking for incremental deltas. + set_presence(str): What state the device presence should be set to. + default is "online". + backfill(bool): Should the HS request message history from other + servers. This may take a long time making it unsuitable for clients + expecting a prompt response. Defaults to "true". + filter(filter_id): A filter to apply to the events returned. + filter_*: Filter override parameters. + + Response JSON:: + { + "next_batch": // batch token for the next /sync + "private_user_data": // private events for this user. + "public_user_data": // public events for all users including the + // public events for this user. + "rooms": [{ // List of rooms with updates. + "room_id": // Id of the room being updated + "limited": // Was the per-room event limit exceeded? + "published": // Is the room published by our HS? + "event_map": // Map of EventID -> event JSON. + "events": { // The recent events in the room if gap is "true" + // otherwise the next events in the room. + "batch": [] // list of EventIDs in the "event_map". + "prev_batch": // back token for getting previous events. + } + "state": [] // list of EventIDs updating the current state to + // be what it should be at the end of the batch. + }] + } + """ + + + PATTERN = client_v2_pattern("/sync$") + ALLOWED_SORT = set(["timeline,asc", "timeline,desc"]) + ALLOWED_PRESENCE = set(["online", "offline", "idle"]) + + def __init__(self, hs): + super(SyncRestServlet, self).__init__() + self.auth = hs.get_auth() + #self.sync_handler = hs.get_handlers().sync_hanlder + + @defer.inlineCallbacks + def on_GET(self, request): + user = yield self.auth.get_user_by_req(request) + + timeout = self.parse_integer(request, "timeout", default=0) + limit = self.parse_integer(request, "limit", default=None) + gap = self.parse_boolean(request, "gap", default=True) + sort = self.parse_string( + request, "sort", default="timeline,asc", + allowed_values=self.ALLOWED_SORT + ) + since = self.parse_string(request, "since") + set_presence = self.parse_string( + request, "set_presence", default="online", + allowed_values=self.ALLOWED_PRESENCE + ) + backfill = self.parse_boolean(request, "backfill", default=True) + filter_id = self.parse_string(request, "filter", default=None) + + logger.info( + "/sync: user=%r, timeout=%r, limit=%r, gap=%r, sort=%r, since=%r," + " set_presence=%r, backfill=%r, filter_id=%r" % ( + user, timeout, limit, gap, sort, since, set_presence, + backfill, filter_id + ) + ) + + # TODO(mjark): Load filter and apply overrides. + # filter = self.filters.load_fitler(filter_id_str) + # filter = filter.apply_overrides(http_request) + # if filter.matches(event): + # # stuff + + # if timeout != 0: + # register for updates from the event stream + + #rooms = [] + + if gap: + pass + # now_stream_token = get_current_stream_token + # for room_id in get_rooms_for_user(user, filter=filter): + # state, events, start, end, limited, published = updates_for_room( + # from=since, to=now_stream_token, limit=limit, + # anchor_to_start=False + # ) + # rooms[room_id] = (state, events, start, limited, published) + # next_stream_token = now. + else: + pass + # now_stream_token = get_current_stream_token + # for room_id in get_rooms_for_user(user, filter=filter) + # state, events, start, end, limited, published = updates_for_room( + # from=since, to=now_stream_token, limit=limit, + # anchor_to_start=False + # ) + # next_stream_token = min(next_stream_token, end) + + + response_content = {} + + defer.returnValue((200, response_content)) + + +def register_servlets(hs, http_server): + SyncRestServlet(hs).register(http_server) -- cgit 1.4.1 From 2b1799883db3facf63b7982a0302542082414974 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 18:49:05 +0000 Subject: Add client v2_alpha resource to synapse server resource tree --- synapse/app/homeserver.py | 7 ++++++- synapse/server.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fabe8ddacb..40d28dcbdc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -32,12 +32,13 @@ from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory from synapse.util.logcontext import LoggingContext from synapse.rest.client.v1 import ClientV1RestResource +from synapse.rest.client.v2_alpha import ClientV2AlphaRestResource from daemonize import Daemonize import twisted.manhole.telnet @@ -62,6 +63,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_client(self): return ClientV1RestResource(self) + def build_resource_for_client_v2_alpha(self): + return ClientV2AlphaRestResource(self) + def build_resource_for_federation(self): return JsonResource() @@ -105,6 +109,7 @@ class SynapseHomeServer(HomeServer): # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] desired_tree = [ (CLIENT_PREFIX, self.get_resource_for_client()), + (CLIENT_V2_ALPHA_PREFIX, self.get_resource_for_client_v2_alpha()), (FEDERATION_PREFIX, self.get_resource_for_federation()), (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), diff --git a/synapse/server.py b/synapse/server.py index 32013b1a91..92ed2c5e32 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -70,6 +70,7 @@ class BaseHomeServer(object): 'notifier', 'distributor', 'resource_for_client', + 'resource_for_client_v2_alpha', 'resource_for_federation', 'resource_for_web_client', 'resource_for_content_repo', -- cgit 1.4.1 From 7b814d3f7fc8137426bc97fb80751753eb8eb94b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 18:54:51 +0000 Subject: Add client v2_alpha resource to synapse server resource tree --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 7 +++- synapse/http/servlet.py | 57 ++++++++++++++++++++++++++++++++ synapse/rest/client/v2_alpha/__init__.py | 29 ++++++++++++++++ synapse/rest/client/v2_alpha/_base.py | 38 +++++++++++++++++++++ synapse/server.py | 1 + 6 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/__init__.py create mode 100644 synapse/rest/client/v2_alpha/_base.py (limited to 'synapse') diff --git a/synapse/api/urls.py b/synapse/api/urls.py index a299392049..693c0efda6 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -16,6 +16,7 @@ """Contains the URL paths to prefix various aspects of the server with. """ CLIENT_PREFIX = "/_matrix/client/api/v1" +CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha" FEDERATION_PREFIX = "/_matrix/federation/v1" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fabe8ddacb..40d28dcbdc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -32,12 +32,13 @@ from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory from synapse.util.logcontext import LoggingContext from synapse.rest.client.v1 import ClientV1RestResource +from synapse.rest.client.v2_alpha import ClientV2AlphaRestResource from daemonize import Daemonize import twisted.manhole.telnet @@ -62,6 +63,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_client(self): return ClientV1RestResource(self) + def build_resource_for_client_v2_alpha(self): + return ClientV2AlphaRestResource(self) + def build_resource_for_federation(self): return JsonResource() @@ -105,6 +109,7 @@ class SynapseHomeServer(HomeServer): # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] desired_tree = [ (CLIENT_PREFIX, self.get_resource_for_client()), + (CLIENT_V2_ALPHA_PREFIX, self.get_resource_for_client_v2_alpha()), (FEDERATION_PREFIX, self.get_resource_for_federation()), (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index d5ccf2742f..a4eb6c817c 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -15,6 +15,8 @@ """ This module contains base REST classes for constructing REST servlets. """ +from synapse.api.errors import SynapseError + import logging @@ -54,3 +56,58 @@ class RestServlet(object): http_server.register_path(method, pattern, method_handler) else: raise NotImplementedError("RestServlet must register something.") + + @staticmethod + def parse_integer(request, name, default=None, required=False): + if name in request.args: + try: + return int(request.args[name][0]) + except: + message = "Query parameter %r must be an integer" % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing integer query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_boolean(request, name, default=None, required=False): + if name in request.args: + try: + return { + "true": True, + "false": False, + }[request.args[name][0]] + except: + message = ( + "Boolean query parameter %r must be one of" + " ['true', 'false']" + ) % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing boolean query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_string(request, name, default=None, required=False, + allowed_values=None, param_type="string"): + if name in request.args: + value = request.args[name][0] + if allowed_values is not None and value not in allowed_values: + message = "Query parameter %r must be one of [%s]" % ( + name, ", ".join(repr(v) for v in allowed_values) + ) + raise SynapseError(message) + else: + return value + else: + if required: + message = "Missing %s query parameter %r" % (param_type, name) + raise SynapseError(400, message) + else: + return default diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py new file mode 100644 index 0000000000..bb740e2803 --- /dev/null +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from synapse.http.server import JsonResource + + +class ClientV2AlphaRestResource(JsonResource): + """A resource for version 2 alpha of the matrix client API.""" + + def __init__(self, hs): + JsonResource.__init__(self) + self.register_servlets(self, hs) + + @staticmethod + def register_servlets(client_resource, hs): + pass diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py new file mode 100644 index 0000000000..22dc5cb862 --- /dev/null +++ b/synapse/rest/client/v2_alpha/_base.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def client_v2_pattern(path_regex): + """Creates a regex compiled client path with the correct client path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) diff --git a/synapse/server.py b/synapse/server.py index 32013b1a91..92ed2c5e32 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -70,6 +70,7 @@ class BaseHomeServer(object): 'notifier', 'distributor', 'resource_for_client', + 'resource_for_client_v2_alpha', 'resource_for_federation', 'resource_for_web_client', 'resource_for_content_repo', -- cgit 1.4.1 From 7b8861924130821c1bbd05ce65260209a993f759 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Jan 2015 10:45:24 +0000 Subject: Split up replication_layer module into client, server and transaction queue --- synapse/federation/federation_client.py | 293 +++++++++++++++ synapse/federation/federation_server.py | 345 ++++++++++++++++++ synapse/federation/replication.py | 608 +------------------------------- synapse/federation/transaction_queue.py | 9 +- synapse/storage/__init__.py | 2 +- 5 files changed, 654 insertions(+), 603 deletions(-) create mode 100644 synapse/federation/federation_client.py create mode 100644 synapse/federation/federation_server.py (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py new file mode 100644 index 0000000000..c80f4c61bc --- /dev/null +++ b/synapse/federation/federation_client.py @@ -0,0 +1,293 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from .units import Edu + +from synapse.util.logutils import log_function +from synapse.events import FrozenEvent + +import logging + + +logger = logging.getLogger(__name__) + + +class FederationClient(object): + @log_function + def send_pdu(self, pdu, destinations): + """Informs the replication layer about a new PDU generated within the + home server that should be transmitted to others. + + TODO: Figure out when we should actually resolve the deferred. + + Args: + pdu (Pdu): The new Pdu. + + Returns: + Deferred: Completes when we have successfully processed the PDU + and replicated it to any interested remote home servers. + """ + order = self._order + self._order += 1 + + logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.event_id) + + # TODO, add errback, etc. + self._transaction_queue.enqueue_pdu(pdu, destinations, order) + + logger.debug( + "[%s] transaction_layer.enqueue_pdu... done", + pdu.event_id + ) + + @log_function + def send_edu(self, destination, edu_type, content): + edu = Edu( + origin=self.server_name, + destination=destination, + edu_type=edu_type, + content=content, + ) + + # TODO, add errback, etc. + self._transaction_queue.enqueue_edu(edu) + return defer.succeed(None) + + @log_function + def send_failure(self, failure, destination): + self._transaction_queue.enqueue_failure(failure, destination) + return defer.succeed(None) + + @log_function + def make_query(self, destination, query_type, args, + retry_on_dns_fail=True): + """Sends a federation Query to a remote homeserver of the given type + and arguments. + + Args: + destination (str): Domain name of the remote homeserver + query_type (str): Category of the query type; should match the + handler name used in register_query_handler(). + args (dict): Mapping of strings to strings containing the details + of the query request. + + Returns: + a Deferred which will eventually yield a JSON object from the + response + """ + return self.transport_layer.make_query( + destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail + ) + + @defer.inlineCallbacks + @log_function + def backfill(self, dest, context, limit, extremities): + """Requests some more historic PDUs for the given context from the + given destination server. + + Args: + dest (str): The remote home server to ask. + context (str): The context to backfill. + limit (int): The maximum number of PDUs to return. + extremities (list): List of PDU id and origins of the first pdus + we have seen from the context + + Returns: + Deferred: Results in the received PDUs. + """ + logger.debug("backfill extrem=%s", extremities) + + # If there are no extremeties then we've (probably) reached the start. + if not extremities: + return + + transaction_data = yield self.transport_layer.backfill( + dest, context, extremities, limit) + + logger.debug("backfill transaction_data=%s", repr(transaction_data)) + + pdus = [ + self.event_from_pdu_json(p, outlier=False) + for p in transaction_data["pdus"] + ] + + defer.returnValue(pdus) + + @defer.inlineCallbacks + @log_function + def get_pdu(self, destinations, event_id, outlier=False): + """Requests the PDU with given origin and ID from the remote home + servers. + + Will attempt to get the PDU from each destination in the list until + one succeeds. + + This will persist the PDU locally upon receipt. + + Args: + destinations (list): Which home servers to query + pdu_origin (str): The home server that originally sent the pdu. + event_id (str) + outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if + it's from an arbitary point in the context as opposed to part + of the current block of PDUs. Defaults to `False` + + Returns: + Deferred: Results in the requested PDU. + """ + + # TODO: Rate limit the number of times we try and get the same event. + + pdu = None + for destination in destinations: + try: + transaction_data = yield self.transport_layer.get_event( + destination, event_id + ) + except Exception as e: + logger.info( + "Failed to get PDU %s from %s because %s", + event_id, destination, e, + ) + continue + + logger.debug("transaction_data %r", transaction_data) + + pdu_list = [ + self.event_from_pdu_json(p, outlier=outlier) + for p in transaction_data["pdus"] + ] + + if pdu_list: + pdu = pdu_list[0] + # TODO: We need to check signatures here + break + + defer.returnValue(pdu) + + @defer.inlineCallbacks + @log_function + def get_state_for_room(self, destination, room_id, event_id): + """Requests all of the `current` state PDUs for a given room from + a remote home server. + + Args: + destination (str): The remote homeserver to query for the state. + room_id (str): The id of the room we're interested in. + event_id (str): The id of the event we want the state at. + + Returns: + Deferred: Results in a list of PDUs. + """ + + result = yield self.transport_layer.get_room_state( + destination, room_id, event_id=event_id, + ) + + pdus = [ + self.event_from_pdu_json(p, outlier=True) for p in result["pdus"] + ] + + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in result.get("auth_chain", []) + ] + + defer.returnValue((pdus, auth_chain)) + + @defer.inlineCallbacks + @log_function + def get_event_auth(self, destination, room_id, event_id): + res = yield self.transport_layer.get_event_auth( + destination, room_id, event_id, + ) + + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in res["auth_chain"] + ] + + auth_chain.sort(key=lambda e: e.depth) + + defer.returnValue(auth_chain) + + @defer.inlineCallbacks + def make_join(self, destination, room_id, user_id): + ret = yield self.transport_layer.make_join( + destination, room_id, user_id + ) + + pdu_dict = ret["event"] + + logger.debug("Got response to make_join: %s", pdu_dict) + + defer.returnValue(self.event_from_pdu_json(pdu_dict)) + + @defer.inlineCallbacks + def send_join(self, destination, pdu): + time_now = self._clock.time_msec() + _, content = yield self.transport_layer.send_join( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=pdu.get_pdu_json(time_now), + ) + + logger.debug("Got content: %s", content) + + state = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("state", []) + ] + + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("auth_chain", []) + ] + + auth_chain.sort(key=lambda e: e.depth) + + defer.returnValue({ + "state": state, + "auth_chain": auth_chain, + }) + + @defer.inlineCallbacks + def send_invite(self, destination, room_id, event_id, pdu): + time_now = self._clock.time_msec() + code, content = yield self.transport_layer.send_invite( + destination=destination, + room_id=room_id, + event_id=event_id, + content=pdu.get_pdu_json(time_now), + ) + + pdu_dict = content["event"] + + logger.debug("Got response to send_invite: %s", pdu_dict) + + defer.returnValue(self.event_from_pdu_json(pdu_dict)) + + def event_from_pdu_json(self, pdu_json, outlier=False): + event = FrozenEvent( + pdu_json + ) + + event.internal_metadata.outlier = outlier + + return event diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py new file mode 100644 index 0000000000..0597725ce7 --- /dev/null +++ b/synapse/federation/federation_server.py @@ -0,0 +1,345 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from .units import Transaction, Edu + +from synapse.util.logutils import log_function +from synapse.util.logcontext import PreserveLoggingContext +from synapse.events import FrozenEvent + +import logging + + +logger = logging.getLogger(__name__) + + +class FederationServer(object): + def set_handler(self, handler): + """Sets the handler that the replication layer will use to communicate + receipt of new PDUs from other home servers. The required methods are + documented on :py:class:`.ReplicationHandler`. + """ + self.handler = handler + + def register_edu_handler(self, edu_type, handler): + if edu_type in self.edu_handlers: + raise KeyError("Already have an EDU handler for %s" % (edu_type,)) + + self.edu_handlers[edu_type] = handler + + def register_query_handler(self, query_type, handler): + """Sets the handler callable that will be used to handle an incoming + federation Query of the given type. + + Args: + query_type (str): Category name of the query, which should match + the string used by make_query. + handler (callable): Invoked to handle incoming queries of this type + + handler is invoked as: + result = handler(args) + + where 'args' is a dict mapping strings to strings of the query + arguments. It should return a Deferred that will eventually yield an + object to encode as JSON. + """ + if query_type in self.query_handlers: + raise KeyError( + "Already have a Query handler for %s" % (query_type,) + ) + + self.query_handlers[query_type] = handler + + @defer.inlineCallbacks + @log_function + def on_backfill_request(self, origin, room_id, versions, limit): + pdus = yield self.handler.on_backfill_request( + origin, room_id, versions, limit + ) + + defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) + + @defer.inlineCallbacks + @log_function + def on_incoming_transaction(self, transaction_data): + transaction = Transaction(**transaction_data) + + for p in transaction.pdus: + if "unsigned" in p: + unsigned = p["unsigned"] + if "age" in unsigned: + p["age"] = unsigned["age"] + if "age" in p: + p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) + del p["age"] + + pdu_list = [ + self.event_from_pdu_json(p) for p in transaction.pdus + ] + + logger.debug("[%s] Got transaction", transaction.transaction_id) + + response = yield self.transaction_actions.have_responded(transaction) + + if response: + logger.debug("[%s] We've already responed to this request", + transaction.transaction_id) + defer.returnValue(response) + return + + logger.debug("[%s] Transaction is new", transaction.transaction_id) + + with PreserveLoggingContext(): + dl = [] + for pdu in pdu_list: + dl.append(self._handle_new_pdu(transaction.origin, pdu)) + + if hasattr(transaction, "edus"): + for edu in [Edu(**x) for x in transaction.edus]: + self.received_edu( + transaction.origin, + edu.edu_type, + edu.content + ) + + results = yield defer.DeferredList(dl) + + ret = [] + for r in results: + if r[0]: + ret.append({}) + else: + logger.exception(r[1]) + ret.append({"error": str(r[1])}) + + logger.debug("Returning: %s", str(ret)) + + yield self.transaction_actions.set_response( + transaction, + 200, response + ) + defer.returnValue((200, response)) + + def received_edu(self, origin, edu_type, content): + if edu_type in self.edu_handlers: + self.edu_handlers[edu_type](origin, content) + else: + logger.warn("Received EDU of type %s with no handler", edu_type) + + @defer.inlineCallbacks + @log_function + def on_context_state_request(self, origin, room_id, event_id): + if event_id: + pdus = yield self.handler.get_state_for_pdu( + origin, room_id, event_id, + ) + auth_chain = yield self.store.get_auth_chain( + [pdu.event_id for pdu in pdus] + ) + else: + raise NotImplementedError("Specify an event") + + defer.returnValue((200, { + "pdus": [pdu.get_pdu_json() for pdu in pdus], + "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain], + })) + + @defer.inlineCallbacks + @log_function + def on_pdu_request(self, origin, event_id): + pdu = yield self._get_persisted_pdu(origin, event_id) + + if pdu: + defer.returnValue( + (200, self._transaction_from_pdus([pdu]).get_dict()) + ) + else: + defer.returnValue((404, "")) + + @defer.inlineCallbacks + @log_function + def on_pull_request(self, origin, versions): + raise NotImplementedError("Pull transactions not implemented") + + @defer.inlineCallbacks + def on_query_request(self, query_type, args): + if query_type in self.query_handlers: + response = yield self.query_handlers[query_type](args) + defer.returnValue((200, response)) + else: + defer.returnValue( + (404, "No handler for Query type '%s'" % (query_type,)) + ) + + @defer.inlineCallbacks + def on_make_join_request(self, room_id, user_id): + pdu = yield self.handler.on_make_join_request(room_id, user_id) + time_now = self._clock.time_msec() + defer.returnValue({"event": pdu.get_pdu_json(time_now)}) + + @defer.inlineCallbacks + def on_invite_request(self, origin, content): + pdu = self.event_from_pdu_json(content) + ret_pdu = yield self.handler.on_invite_request(origin, pdu) + time_now = self._clock.time_msec() + defer.returnValue((200, {"event": ret_pdu.get_pdu_json(time_now)})) + + @defer.inlineCallbacks + def on_send_join_request(self, origin, content): + logger.debug("on_send_join_request: content: %s", content) + pdu = self.event_from_pdu_json(content) + logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures) + res_pdus = yield self.handler.on_send_join_request(origin, pdu) + time_now = self._clock.time_msec() + defer.returnValue((200, { + "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]], + "auth_chain": [ + p.get_pdu_json(time_now) for p in res_pdus["auth_chain"] + ], + })) + + @defer.inlineCallbacks + def on_event_auth(self, origin, room_id, event_id): + time_now = self._clock.time_msec() + auth_pdus = yield self.handler.on_event_auth(event_id) + defer.returnValue((200, { + "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus], + })) + + @log_function + def _get_persisted_pdu(self, origin, event_id, do_auth=True): + """ Get a PDU from the database with given origin and id. + + Returns: + Deferred: Results in a `Pdu`. + """ + return self.handler.get_persisted_pdu( + origin, event_id, do_auth=do_auth + ) + + def _transaction_from_pdus(self, pdu_list): + """Returns a new Transaction containing the given PDUs suitable for + transmission. + """ + time_now = self._clock.time_msec() + pdus = [p.get_pdu_json(time_now) for p in pdu_list] + return Transaction( + origin=self.server_name, + pdus=pdus, + origin_server_ts=int(time_now), + destination=None, + ) + + @defer.inlineCallbacks + @log_function + def _handle_new_pdu(self, origin, pdu, max_recursion=10): + # We reprocess pdus when we have seen them only as outliers + existing = yield self._get_persisted_pdu( + origin, pdu.event_id, do_auth=False + ) + + already_seen = ( + existing and ( + not existing.internal_metadata.is_outlier() + or pdu.internal_metadata.is_outlier() + ) + ) + if already_seen: + logger.debug("Already seen pdu %s", pdu.event_id) + defer.returnValue({}) + return + + state = None + + auth_chain = [] + + have_seen = yield self.store.have_events( + [e for e, _ in pdu.prev_events] + ) + + # Get missing pdus if necessary. + if not pdu.internal_metadata.is_outlier(): + # We only backfill backwards to the min depth. + min_depth = yield self.handler.get_min_depth_for_context( + pdu.room_id + ) + + logger.debug( + "_handle_new_pdu min_depth for %s: %d", + pdu.room_id, min_depth + ) + + if min_depth and pdu.depth > min_depth and max_recursion > 0: + for event_id, hashes in pdu.prev_events: + if event_id not in have_seen: + logger.debug( + "_handle_new_pdu requesting pdu %s", + event_id + ) + + try: + new_pdu = yield self.federation_client.get_pdu( + [origin, pdu.origin], + event_id=event_id, + ) + + if new_pdu: + yield self._handle_new_pdu( + origin, + new_pdu, + max_recursion=max_recursion-1 + ) + + logger.debug("Processed pdu %s", event_id) + else: + logger.warn("Failed to get PDU %s", event_id) + except: + # TODO(erikj): Do some more intelligent retries. + logger.exception("Failed to get PDU") + else: + # We need to get the state at this event, since we have reached + # a backward extremity edge. + logger.debug( + "_handle_new_pdu getting state for %s", + pdu.room_id + ) + state, auth_chain = yield self.get_state_for_room( + origin, pdu.room_id, pdu.event_id, + ) + + ret = yield self.handler.on_receive_pdu( + origin, + pdu, + backfilled=False, + state=state, + auth_chain=auth_chain, + ) + + defer.returnValue(ret) + + def __str__(self): + return "" % self.server_name + + def event_from_pdu_json(self, pdu_json, outlier=False): + event = FrozenEvent( + pdu_json + ) + + event.internal_metadata.outlier = outlier + + return event diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index accf95e406..9ef4834927 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -17,15 +17,12 @@ a given transport. """ -from twisted.internet import defer +from .federation_client import FederationClient +from .federation_server import FederationServer -from .units import Transaction, Edu -from .persistence import TransactionActions from .transaction_queue import TransactionQueue -from synapse.util.logutils import log_function -from synapse.util.logcontext import PreserveLoggingContext -from synapse.events import FrozenEvent +from .persistence import TransactionActions import logging @@ -33,7 +30,7 @@ import logging logger = logging.getLogger(__name__) -class ReplicationLayer(object): +class ReplicationLayer(FederationClient, FederationServer): """This layer is responsible for replicating with remote home servers over the given transport. I.e., does the sending and receiving of PDUs to remote home servers. @@ -58,607 +55,20 @@ class ReplicationLayer(object): self.transport_layer.register_received_handler(self) self.transport_layer.register_request_handler(self) - self.store = hs.get_datastore() - # self.pdu_actions = PduActions(self.store) - self.transaction_actions = TransactionActions(self.store) + self.federation_client = self - self._transaction_queue = TransactionQueue( - hs, self.transaction_actions, transport_layer - ) + self.store = hs.get_datastore() self.handler = None self.edu_handlers = {} self.query_handlers = {} - self._order = 0 - self._clock = hs.get_clock() - self.event_builder_factory = hs.get_event_builder_factory() - - def set_handler(self, handler): - """Sets the handler that the replication layer will use to communicate - receipt of new PDUs from other home servers. The required methods are - documented on :py:class:`.ReplicationHandler`. - """ - self.handler = handler - - def register_edu_handler(self, edu_type, handler): - if edu_type in self.edu_handlers: - raise KeyError("Already have an EDU handler for %s" % (edu_type,)) - - self.edu_handlers[edu_type] = handler - - def register_query_handler(self, query_type, handler): - """Sets the handler callable that will be used to handle an incoming - federation Query of the given type. - - Args: - query_type (str): Category name of the query, which should match - the string used by make_query. - handler (callable): Invoked to handle incoming queries of this type - - handler is invoked as: - result = handler(args) - - where 'args' is a dict mapping strings to strings of the query - arguments. It should return a Deferred that will eventually yield an - object to encode as JSON. - """ - if query_type in self.query_handlers: - raise KeyError( - "Already have a Query handler for %s" % (query_type,) - ) - - self.query_handlers[query_type] = handler - - @log_function - def send_pdu(self, pdu, destinations): - """Informs the replication layer about a new PDU generated within the - home server that should be transmitted to others. - - TODO: Figure out when we should actually resolve the deferred. - - Args: - pdu (Pdu): The new Pdu. - - Returns: - Deferred: Completes when we have successfully processed the PDU - and replicated it to any interested remote home servers. - """ - order = self._order - self._order += 1 - - logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.event_id) - - # TODO, add errback, etc. - self._transaction_queue.enqueue_pdu(pdu, destinations, order) - - logger.debug( - "[%s] transaction_layer.enqueue_pdu... done", - pdu.event_id - ) - - @log_function - def send_edu(self, destination, edu_type, content): - edu = Edu( - origin=self.server_name, - destination=destination, - edu_type=edu_type, - content=content, - ) - - # TODO, add errback, etc. - self._transaction_queue.enqueue_edu(edu) - return defer.succeed(None) - - @log_function - def send_failure(self, failure, destination): - self._transaction_queue.enqueue_failure(failure, destination) - return defer.succeed(None) - - @log_function - def make_query(self, destination, query_type, args, - retry_on_dns_fail=True): - """Sends a federation Query to a remote homeserver of the given type - and arguments. - - Args: - destination (str): Domain name of the remote homeserver - query_type (str): Category of the query type; should match the - handler name used in register_query_handler(). - args (dict): Mapping of strings to strings containing the details - of the query request. - - Returns: - a Deferred which will eventually yield a JSON object from the - response - """ - return self.transport_layer.make_query( - destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail - ) - - @defer.inlineCallbacks - @log_function - def backfill(self, dest, context, limit, extremities): - """Requests some more historic PDUs for the given context from the - given destination server. - - Args: - dest (str): The remote home server to ask. - context (str): The context to backfill. - limit (int): The maximum number of PDUs to return. - extremities (list): List of PDU id and origins of the first pdus - we have seen from the context - - Returns: - Deferred: Results in the received PDUs. - """ - logger.debug("backfill extrem=%s", extremities) - - # If there are no extremeties then we've (probably) reached the start. - if not extremities: - return - - transaction_data = yield self.transport_layer.backfill( - dest, context, extremities, limit) - - logger.debug("backfill transaction_data=%s", repr(transaction_data)) - - transaction = Transaction(**transaction_data) - - pdus = [ - self.event_from_pdu_json(p, outlier=False) - for p in transaction.pdus - ] - for pdu in pdus: - yield self._handle_new_pdu(dest, pdu, backfilled=True) - - defer.returnValue(pdus) - - @defer.inlineCallbacks - @log_function - def get_pdu(self, destination, event_id, outlier=False): - """Requests the PDU with given origin and ID from the remote home - server. - - This will persist the PDU locally upon receipt. - - Args: - destination (str): Which home server to query - pdu_origin (str): The home server that originally sent the pdu. - event_id (str) - outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if - it's from an arbitary point in the context as opposed to part - of the current block of PDUs. Defaults to `False` - - Returns: - Deferred: Results in the requested PDU. - """ - - transaction_data = yield self.transport_layer.get_event( - destination, event_id - ) - - transaction = Transaction(**transaction_data) - - pdu_list = [ - self.event_from_pdu_json(p, outlier=outlier) - for p in transaction.pdus - ] - - pdu = None - if pdu_list: - pdu = pdu_list[0] - yield self._handle_new_pdu(destination, pdu) - - defer.returnValue(pdu) - - @defer.inlineCallbacks - @log_function - def get_state_for_room(self, destination, room_id, event_id): - """Requests all of the `current` state PDUs for a given room from - a remote home server. - - Args: - destination (str): The remote homeserver to query for the state. - room_id (str): The id of the room we're interested in. - event_id (str): The id of the event we want the state at. - - Returns: - Deferred: Results in a list of PDUs. - """ - - result = yield self.transport_layer.get_room_state( - destination, room_id, event_id=event_id, - ) - - pdus = [ - self.event_from_pdu_json(p, outlier=True) for p in result["pdus"] - ] - - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in result.get("auth_chain", []) - ] - - defer.returnValue((pdus, auth_chain)) - - @defer.inlineCallbacks - @log_function - def get_event_auth(self, destination, room_id, event_id): - res = yield self.transport_layer.get_event_auth( - destination, room_id, event_id, - ) - - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in res["auth_chain"] - ] - - auth_chain.sort(key=lambda e: e.depth) - - defer.returnValue(auth_chain) - - @defer.inlineCallbacks - @log_function - def on_backfill_request(self, origin, room_id, versions, limit): - pdus = yield self.handler.on_backfill_request( - origin, room_id, versions, limit - ) - - defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) - - @defer.inlineCallbacks - @log_function - def on_incoming_transaction(self, transaction_data): - transaction = Transaction(**transaction_data) - - for p in transaction.pdus: - if "unsigned" in p: - unsigned = p["unsigned"] - if "age" in unsigned: - p["age"] = unsigned["age"] - if "age" in p: - p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) - del p["age"] - - pdu_list = [ - self.event_from_pdu_json(p) for p in transaction.pdus - ] - - logger.debug("[%s] Got transaction", transaction.transaction_id) - - response = yield self.transaction_actions.have_responded(transaction) - - if response: - logger.debug("[%s] We've already responed to this request", - transaction.transaction_id) - defer.returnValue(response) - return - - logger.debug("[%s] Transaction is new", transaction.transaction_id) - - with PreserveLoggingContext(): - dl = [] - for pdu in pdu_list: - dl.append(self._handle_new_pdu(transaction.origin, pdu)) - - if hasattr(transaction, "edus"): - for edu in [Edu(**x) for x in transaction.edus]: - self.received_edu( - transaction.origin, - edu.edu_type, - edu.content - ) - - results = yield defer.DeferredList(dl) - - ret = [] - for r in results: - if r[0]: - ret.append({}) - else: - logger.exception(r[1]) - ret.append({"error": str(r[1])}) - - logger.debug("Returning: %s", str(ret)) - - yield self.transaction_actions.set_response( - transaction, - 200, response - ) - defer.returnValue((200, response)) - - def received_edu(self, origin, edu_type, content): - if edu_type in self.edu_handlers: - self.edu_handlers[edu_type](origin, content) - else: - logger.warn("Received EDU of type %s with no handler", edu_type) - - @defer.inlineCallbacks - @log_function - def on_context_state_request(self, origin, room_id, event_id): - if event_id: - pdus = yield self.handler.get_state_for_pdu( - origin, room_id, event_id, - ) - auth_chain = yield self.store.get_auth_chain( - [pdu.event_id for pdu in pdus] - ) - else: - raise NotImplementedError("Specify an event") - - defer.returnValue((200, { - "pdus": [pdu.get_pdu_json() for pdu in pdus], - "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain], - })) - - @defer.inlineCallbacks - @log_function - def on_pdu_request(self, origin, event_id): - pdu = yield self._get_persisted_pdu(origin, event_id) - - if pdu: - defer.returnValue( - (200, self._transaction_from_pdus([pdu]).get_dict()) - ) - else: - defer.returnValue((404, "")) - - @defer.inlineCallbacks - @log_function - def on_pull_request(self, origin, versions): - raise NotImplementedError("Pull transactions not implemented") - - @defer.inlineCallbacks - def on_query_request(self, query_type, args): - if query_type in self.query_handlers: - response = yield self.query_handlers[query_type](args) - defer.returnValue((200, response)) - else: - defer.returnValue( - (404, "No handler for Query type '%s'" % (query_type,)) - ) - - @defer.inlineCallbacks - def on_make_join_request(self, room_id, user_id): - pdu = yield self.handler.on_make_join_request(room_id, user_id) - time_now = self._clock.time_msec() - defer.returnValue({"event": pdu.get_pdu_json(time_now)}) - - @defer.inlineCallbacks - def on_invite_request(self, origin, content): - pdu = self.event_from_pdu_json(content) - ret_pdu = yield self.handler.on_invite_request(origin, pdu) - time_now = self._clock.time_msec() - defer.returnValue((200, {"event": ret_pdu.get_pdu_json(time_now)})) - - @defer.inlineCallbacks - def on_send_join_request(self, origin, content): - logger.debug("on_send_join_request: content: %s", content) - pdu = self.event_from_pdu_json(content) - logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures) - res_pdus = yield self.handler.on_send_join_request(origin, pdu) - time_now = self._clock.time_msec() - defer.returnValue((200, { - "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]], - "auth_chain": [ - p.get_pdu_json(time_now) for p in res_pdus["auth_chain"] - ], - })) - - @defer.inlineCallbacks - def on_event_auth(self, origin, room_id, event_id): - time_now = self._clock.time_msec() - auth_pdus = yield self.handler.on_event_auth(event_id) - defer.returnValue((200, { - "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus], - })) - - @defer.inlineCallbacks - def make_join(self, destination, room_id, user_id): - ret = yield self.transport_layer.make_join( - destination, room_id, user_id - ) - - pdu_dict = ret["event"] - - logger.debug("Got response to make_join: %s", pdu_dict) - - defer.returnValue(self.event_from_pdu_json(pdu_dict)) - - @defer.inlineCallbacks - def send_join(self, destination, pdu): - time_now = self._clock.time_msec() - _, content = yield self.transport_layer.send_join( - destination=destination, - room_id=pdu.room_id, - event_id=pdu.event_id, - content=pdu.get_pdu_json(time_now), - ) - - logger.debug("Got content: %s", content) - - state = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("state", []) - ] - - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("auth_chain", []) - ] - - auth_chain.sort(key=lambda e: e.depth) - - defer.returnValue({ - "state": state, - "auth_chain": auth_chain, - }) - - @defer.inlineCallbacks - def send_invite(self, destination, room_id, event_id, pdu): - time_now = self._clock.time_msec() - code, content = yield self.transport_layer.send_invite( - destination=destination, - room_id=room_id, - event_id=event_id, - content=pdu.get_pdu_json(time_now), - ) - - pdu_dict = content["event"] - - logger.debug("Got response to send_invite: %s", pdu_dict) - - defer.returnValue(self.event_from_pdu_json(pdu_dict)) - - @log_function - def _get_persisted_pdu(self, origin, event_id, do_auth=True): - """ Get a PDU from the database with given origin and id. - - Returns: - Deferred: Results in a `Pdu`. - """ - return self.handler.get_persisted_pdu( - origin, event_id, do_auth=do_auth - ) - - def _transaction_from_pdus(self, pdu_list): - """Returns a new Transaction containing the given PDUs suitable for - transmission. - """ - time_now = self._clock.time_msec() - pdus = [p.get_pdu_json(time_now) for p in pdu_list] - return Transaction( - origin=self.server_name, - pdus=pdus, - origin_server_ts=int(time_now), - destination=None, - ) - - @defer.inlineCallbacks - @log_function - def _handle_new_pdu(self, origin, pdu, backfilled=False): - # We reprocess pdus when we have seen them only as outliers - existing = yield self._get_persisted_pdu( - origin, pdu.event_id, do_auth=False - ) - - already_seen = ( - existing and ( - not existing.internal_metadata.is_outlier() - or pdu.internal_metadata.is_outlier() - ) - ) - if already_seen: - logger.debug("Already seen pdu %s", pdu.event_id) - defer.returnValue({}) - return - - state = None - - auth_chain = [] - - # We need to make sure we have all the auth events. - # for e_id, _ in pdu.auth_events: - # exists = yield self._get_persisted_pdu( - # origin, - # e_id, - # do_auth=False - # ) - # - # if not exists: - # try: - # logger.debug( - # "_handle_new_pdu fetch missing auth event %s from %s", - # e_id, - # origin, - # ) - # - # yield self.get_pdu( - # origin, - # event_id=e_id, - # outlier=True, - # ) - # - # logger.debug("Processed pdu %s", e_id) - # except: - # logger.warn( - # "Failed to get auth event %s from %s", - # e_id, - # origin - # ) - - # Get missing pdus if necessary. - if not pdu.internal_metadata.is_outlier(): - # We only backfill backwards to the min depth. - min_depth = yield self.handler.get_min_depth_for_context( - pdu.room_id - ) - - logger.debug( - "_handle_new_pdu min_depth for %s: %d", - pdu.room_id, min_depth - ) - - if min_depth and pdu.depth > min_depth: - for event_id, hashes in pdu.prev_events: - exists = yield self._get_persisted_pdu( - origin, - event_id, - do_auth=False - ) - - if not exists: - logger.debug( - "_handle_new_pdu requesting pdu %s", - event_id - ) - - try: - yield self.get_pdu( - origin, - event_id=event_id, - ) - logger.debug("Processed pdu %s", event_id) - except: - # TODO(erikj): Do some more intelligent retries. - logger.exception("Failed to get PDU") - else: - # We need to get the state at this event, since we have reached - # a backward extremity edge. - logger.debug( - "_handle_new_pdu getting state for %s", - pdu.room_id - ) - state, auth_chain = yield self.get_state_for_room( - origin, pdu.room_id, pdu.event_id, - ) - - if not backfilled: - ret = yield self.handler.on_receive_pdu( - origin, - pdu, - backfilled=backfilled, - state=state, - auth_chain=auth_chain, - ) - else: - ret = None - - # yield self.pdu_actions.mark_as_processed(pdu) + self.transaction_actions = TransactionActions(self.store) + self._transaction_queue = TransactionQueue(hs, transport_layer) - defer.returnValue(ret) + self._order = 0 def __str__(self): return "" % self.server_name - - def event_from_pdu_json(self, pdu_json, outlier=False): - event = FrozenEvent( - pdu_json - ) - - event.internal_metadata.outlier = outlier - - return event diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index c2cb4a1c49..9d4f2c09a2 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -16,6 +16,7 @@ from twisted.internet import defer +from .persistence import TransactionActions from .units import Transaction from synapse.util.logutils import log_function @@ -34,13 +35,15 @@ class TransactionQueue(object): It batches pending PDUs into single transactions. """ - def __init__(self, hs, transaction_actions, transport_layer): + def __init__(self, hs, transport_layer): self.server_name = hs.hostname - self.transaction_actions = transaction_actions + + self.store = hs.get_datastore() + self.transaction_actions = TransactionActions(self.store) + self.transport_layer = transport_layer self._clock = hs.get_clock() - self.store = hs.get_datastore() # Is a mapping from destinations -> deferreds. Used to keep track # of which destinations have transactions in flight and when they are diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4f09909607..27d835db79 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -434,7 +434,7 @@ class DataStore(RoomMemberStore, RoomStore, sql = ( "SELECT e.event_id, reason FROM events as e " "LEFT JOIN rejections as r ON e.event_id = r.event_id " - "WHERE event_id = ?" + "WHERE e.event_id = ?" ) res = {} -- cgit 1.4.1 From c92d64a6c35713aabaed11e8ef1e62d2fb84a875 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Jan 2015 14:33:11 +0000 Subject: Make it the responsibility of the replication layer to check signature and hashes. --- synapse/federation/federation_client.py | 108 ++++++++++++++++++++++++++++---- synapse/federation/federation_server.py | 89 ++++++++++++++++++++++---- synapse/federation/replication.py | 2 + 3 files changed, 173 insertions(+), 26 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index c80f4c61bc..91b44cd8b3 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -20,6 +20,13 @@ from .units import Edu from synapse.util.logutils import log_function from synapse.events import FrozenEvent +from synapse.events.utils import prune_event + +from syutil.jsonutil import encode_canonical_json + +from synapse.crypto.event_signing import check_event_content_hash + +from synapse.api.errors import SynapseError import logging @@ -126,6 +133,11 @@ class FederationClient(object): for p in transaction_data["pdus"] ] + for i, pdu in enumerate(pdus): + pdus[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + defer.returnValue(pdus) @defer.inlineCallbacks @@ -159,6 +171,22 @@ class FederationClient(object): transaction_data = yield self.transport_layer.get_event( destination, event_id ) + + logger.debug("transaction_data %r", transaction_data) + + pdu_list = [ + self.event_from_pdu_json(p, outlier=outlier) + for p in transaction_data["pdus"] + ] + + if pdu_list: + pdu = pdu_list[0] + + # Check signatures are correct. + pdu = yield self._check_sigs_and_hash(pdu) + + break + except Exception as e: logger.info( "Failed to get PDU %s from %s because %s", @@ -166,18 +194,6 @@ class FederationClient(object): ) continue - logger.debug("transaction_data %r", transaction_data) - - pdu_list = [ - self.event_from_pdu_json(p, outlier=outlier) - for p in transaction_data["pdus"] - ] - - if pdu_list: - pdu = pdu_list[0] - # TODO: We need to check signatures here - break - defer.returnValue(pdu) @defer.inlineCallbacks @@ -208,6 +224,16 @@ class FederationClient(object): for p in result.get("auth_chain", []) ] + for i, pdu in enumerate(pdus): + pdus[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + + for i, pdu in enumerate(auth_chain): + auth_chain[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + defer.returnValue((pdus, auth_chain)) @defer.inlineCallbacks @@ -222,6 +248,11 @@ class FederationClient(object): for p in res["auth_chain"] ] + for i, pdu in enumerate(auth_chain): + auth_chain[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + auth_chain.sort(key=lambda e: e.depth) defer.returnValue(auth_chain) @@ -260,6 +291,16 @@ class FederationClient(object): for p in content.get("auth_chain", []) ] + for i, pdu in enumerate(state): + state[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + + for i, pdu in enumerate(auth_chain): + auth_chain[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + auth_chain.sort(key=lambda e: e.depth) defer.returnValue({ @@ -281,7 +322,14 @@ class FederationClient(object): logger.debug("Got response to send_invite: %s", pdu_dict) - defer.returnValue(self.event_from_pdu_json(pdu_dict)) + pdu = self.event_from_pdu_json(pdu_dict) + + # Check signatures are correct. + pdu = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + + defer.returnValue(pdu) def event_from_pdu_json(self, pdu_json, outlier=False): event = FrozenEvent( @@ -291,3 +339,37 @@ class FederationClient(object): event.internal_metadata.outlier = outlier return event + + @defer.inlineCallbacks + def _check_sigs_and_hash(self, pdu): + """Throws a SynapseError if the PDU does not have the correct + signatures. + + Returns: + FrozenEvent: Either the given event or it redacted if it failed the + content hash check. + """ + # Check signatures are correct. + redacted_event = prune_event(pdu) + redacted_pdu_json = redacted_event.get_pdu_json() + + try: + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) + except SynapseError: + logger.warn( + "Signature check failed for %s redacted to %s", + encode_canonical_json(pdu.get_pdu_json()), + encode_canonical_json(redacted_pdu_json), + ) + raise + + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s, %s", + pdu.event_id, encode_canonical_json(pdu.get_dict()) + ) + defer.returnValue(redacted_event) + + defer.returnValue(pdu) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 0597725ce7..fc5342afaa 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -21,6 +21,13 @@ from .units import Transaction, Edu from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext from synapse.events import FrozenEvent +from synapse.events.utils import prune_event + +from syutil.jsonutil import encode_canonical_json + +from synapse.crypto.event_signing import check_event_content_hash + +from synapse.api.errors import FederationError, SynapseError import logging @@ -97,8 +104,10 @@ class FederationServer(object): response = yield self.transaction_actions.have_responded(transaction) if response: - logger.debug("[%s] We've already responed to this request", - transaction.transaction_id) + logger.debug( + "[%s] We've already responed to this request", + transaction.transaction_id + ) defer.returnValue(response) return @@ -253,6 +262,9 @@ class FederationServer(object): origin, pdu.event_id, do_auth=False ) + # FIXME: Currently we fetch an event again when we already have it + # if it has been marked as an outlier. + already_seen = ( existing and ( not existing.internal_metadata.is_outlier() @@ -264,14 +276,27 @@ class FederationServer(object): defer.returnValue({}) return + # Check signature. + try: + pdu = yield self._check_sigs_and_hash(pdu) + except SynapseError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=pdu.event_id, + ) + state = None auth_chain = [] have_seen = yield self.store.have_events( - [e for e, _ in pdu.prev_events] + [ev for ev, _ in pdu.prev_events] ) + fetch_state = False + # Get missing pdus if necessary. if not pdu.internal_metadata.is_outlier(): # We only backfill backwards to the min depth. @@ -311,16 +336,20 @@ class FederationServer(object): except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") - else: - # We need to get the state at this event, since we have reached - # a backward extremity edge. - logger.debug( - "_handle_new_pdu getting state for %s", - pdu.room_id - ) - state, auth_chain = yield self.get_state_for_room( - origin, pdu.room_id, pdu.event_id, - ) + fetch_state = True + else: + fetch_state = True + + if fetch_state: + # We need to get the state at this event, since we haven't + # processed all the prev events. + logger.debug( + "_handle_new_pdu getting state for %s", + pdu.room_id + ) + state, auth_chain = yield self.get_state_for_room( + origin, pdu.room_id, pdu.event_id, + ) ret = yield self.handler.on_receive_pdu( origin, @@ -343,3 +372,37 @@ class FederationServer(object): event.internal_metadata.outlier = outlier return event + + @defer.inlineCallbacks + def _check_sigs_and_hash(self, pdu): + """Throws a SynapseError if the PDU does not have the correct + signatures. + + Returns: + FrozenEvent: Either the given event or it redacted if it failed the + content hash check. + """ + # Check signatures are correct. + redacted_event = prune_event(pdu) + redacted_pdu_json = redacted_event.get_pdu_json() + + try: + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) + except SynapseError: + logger.warn( + "Signature check failed for %s redacted to %s", + encode_canonical_json(pdu.get_pdu_json()), + encode_canonical_json(redacted_pdu_json), + ) + raise + + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s, %s", + pdu.event_id, encode_canonical_json(pdu.get_dict()) + ) + defer.returnValue(redacted_event) + + defer.returnValue(pdu) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9ef4834927..e442c6c5d5 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -51,6 +51,8 @@ class ReplicationLayer(FederationClient, FederationServer): def __init__(self, hs, transport_layer): self.server_name = hs.hostname + self.keyring = hs.get_keyring() + self.transport_layer = transport_layer self.transport_layer.register_received_handler(self) self.transport_layer.register_request_handler(self) -- cgit 1.4.1 From 8d7accb28fdf2b8936eb865a716593a2ffcea0ed Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 23 Jan 2015 19:52:30 +0000 Subject: Initial minimal attempt at /user/:user_id/filter API - in-memory storage, no actual filter implementation --- synapse/rest/client/v2_alpha/__init__.py | 7 ++- synapse/rest/client/v2_alpha/filter.py | 103 +++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/filter.py (limited to 'synapse') diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index bb740e2803..4349579ab7 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -14,6 +14,11 @@ # limitations under the License. +from . import ( + filter +) + + from synapse.http.server import JsonResource @@ -26,4 +31,4 @@ class ClientV2AlphaRestResource(JsonResource): @staticmethod def register_servlets(client_resource, hs): - pass + filter.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py new file mode 100644 index 0000000000..a9a180ec04 --- /dev/null +++ b/synapse/rest/client/v2_alpha/filter.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import AuthError, SynapseError +from synapse.http.servlet import RestServlet +from synapse.types import UserID + +from ._base import client_v2_pattern + +import json +import logging + + +logger = logging.getLogger(__name__) + + +# TODO(paul) +_filters_for_user = {} + + +class GetFilterRestServlet(RestServlet): + PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter/(?P[^/]*)") + + def __init__(self, hs): + super(GetFilterRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + + @defer.inlineCallbacks + def on_GET(self, request, user_id, filter_id): + target_user = UserID.from_string(user_id) + auth_user = yield self.auth.get_user_by_req(request) + + if target_user != auth_user: + raise AuthError(403, "Cannot get filters for other users") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only get filters for local users") + + try: + filter_id = int(filter_id) + except: + raise SynapseError(400, "Invalid filter_id") + + filters = _filters_for_user.get(target_user.localpart, None) + + if not filters or filter_id >= len(filters): + raise SynapseError(400, "No such filter") + + defer.returnValue((200, filters[filter_id])) + + +class CreateFilterRestServlet(RestServlet): + PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter") + + def __init__(self, hs): + super(CreateFilterRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + + @defer.inlineCallbacks + def on_POST(self, request, user_id): + target_user = UserID.from_string(user_id) + auth_user = yield self.auth.get_user_by_req(request) + + if target_user != auth_user: + raise AuthError(403, "Cannot create filters for other users") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only create filters for local users") + + try: + content = json.loads(request.content.read()) + + # TODO(paul): check for required keys and invalid keys + except: + raise SynapseError(400, "Invalid filter definition") + + filters = _filters_for_user.setdefault(target_user.localpart, []) + + filter_id = len(filters) + filters.append(content) + + defer.returnValue((200, {"filter_id": str(filter_id)})) + + +def register_servlets(hs, http_server): + GetFilterRestServlet(hs).register(http_server) + CreateFilterRestServlet(hs).register(http_server) -- cgit 1.4.1 From efac71d6caeac9ef39dab521467932892b324e99 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 26 Jan 2015 14:37:14 +0000 Subject: Pushers should only try & look for rejected devices in something that's a list or tuple. --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 53d3319699..0c51d2dd8f 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -173,7 +173,7 @@ class Pusher(object): processed = True else: rejected = yield self.dispatch_push(single_event, tweaks) - if not rejected is False: + if isinstance(rejected, list) or isinstance(rejected, tuple): processed = True for pk in rejected: if pk != self.pushkey: -- cgit 1.4.1 From 0cfb4591a7754bcc08edddd17629006b5096d94d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 15:46:31 +0000 Subject: Add handler for /sync API --- synapse/handlers/sync.py | 110 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 synapse/handlers/sync.py (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py new file mode 100644 index 0000000000..ec20ea4890 --- /dev/null +++ b/synapse/handlers/sync.py @@ -0,0 +1,110 @@ +import collections + + +SyncConfig = collections.namedtuple("SyncConfig", [ + "user", + "device", + "since", + "limit", + "gap", + "sort" + "backfill" + "filter", +) + + +RoomSyncResult = collections.namedtuple("RoomSyncResult", [ + "room_id", + "limited", + "published", + "prev_batch", + "events", + "state", + "event_map", +]) + + +class SyncResult(collections.namedtuple("SyncResult", [ + "next_batch", # Token for the next sync + "private_user_data", # List of private events for the user. + "public_user_data", # List of public events for all users. + "rooms", # RoomSyncResult for each room. +])): + __slots__ = [] + + def __nonzero__(self): + return self.private_user_data or self.public_user_data or self.rooms + + +class SyncHandler(BaseHandler): + + def __init__(self, hs): + super(SyncHandler, self).__init__(hs) + self.event_sources = hs.get_event_sources() + + def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): + if timeout == 0: + return self.current_sync_for_user(sync_config, since) + else: + def current_sync_callback(since_token): + return self.current_sync_for_user( + self, since_token, sync_config + ) + return self.notifier.wait_for_events( + sync_config.filter, since_token, current_sync_callback + ) + defer.returnValue(result) + + def current_sync_for_user(self, sync_config, since_token=None): + if since_token is None: + return self.inital_sync(sync_config) + else: + return self.incremental_sync(sync_config) + + @defer.inlineCallbacks + def initial_sync(self, sync_config): + now_token = yield self.event_sources.get_current_token() + + presence_stream = self.event_sources.sources["presence"] + # TODO (markjh): This looks wrong, shouldn't we be getting the presence + # UP to the present rather than after the present? + pagination_config = PaginationConfig(from_token=now_token) + presence, _ = yield presence_stream.get_pagination_rows( + user, pagination_config.get_source_config("presence"), None + ) + room_list = yield self.store.get_rooms_for_user_where_membership_is( + user_id=user_id, + membership_list=[Membership.INVITE, Membership.JOIN] + ) + + # TODO (markjh): Does public mean "published"? + published_rooms = yield self.store.get_rooms(is_public=True) + published_room_ids = set(r["room_id"] for r in public_rooms) + + for event in room_list: + + messages, token = yield self.store.get_recent_events_for_room( + event.room_id, + limit=sync_config.limit, + end_token=now_token.room_key, + ) + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state = yield self.state_handler.get_current_state( + event.room_id + ) + + rooms.append(RoomSyncResult( + room_id=event.room_id, + published=event.room_id in published_room_ids, + + + + + + @defer.inlineCallbacks + def incremental_sync(self, sync_config): + + + + + -- cgit 1.4.1 From 7f6f3f9d6247076493c4e9d48c8282e25892f8b5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 16:11:28 +0000 Subject: Pass the current time to serialize event, rather than passing an HS and getting a clock from it and calling time_msec on the clock. Remove the serialize_event method from the HS since it is no longer needed. --- synapse/events/utils.py | 12 ++++++------ synapse/handlers/events.py | 5 ++++- synapse/handlers/message.py | 21 +++++++++++++++------ synapse/handlers/room.py | 7 +++++-- synapse/rest/client/v1/events.py | 8 +++++++- synapse/rest/client/v1/room.py | 9 ++++++++- synapse/server.py | 3 --- 7 files changed, 45 insertions(+), 20 deletions(-) (limited to 'synapse') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index bcb5457278..e391aca4cc 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,31 +89,31 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(hs, e, client_event=True): +def serialize_event(e, time_now_ms, client_event=True): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e + time_now_ms = int(time_now_ms) + # Should this strip out None's? d = {k: v for k, v in e.get_dict().items()} if not client_event: # set the age and keep all other keys if "age_ts" in d["unsigned"]: - now = int(hs.get_clock().time_msec()) - d["unsigned"]["age"] = now - d["unsigned"]["age_ts"] + d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] return d if "age_ts" in d["unsigned"]: - now = int(hs.get_clock().time_msec()) - d["age"] = now - d["unsigned"]["age_ts"] + d["age"] = time_now_ms - d["unsigned"]["age_ts"] del d["unsigned"]["age_ts"] d["user_id"] = d.pop("sender", None) if "redacted_because" in e.unsigned: d["redacted_because"] = serialize_event( - hs, e.unsigned["redacted_because"] + e.unsigned["redacted_because"], time_now_ms ) del d["unsigned"]["redacted_because"] diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 01e67b0818..d997917cd6 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logcontext import PreserveLoggingContext from synapse.util.logutils import log_function from synapse.types import UserID +from synapse.events.utils import serialize_event from ._base import BaseHandler @@ -78,8 +79,10 @@ class EventStreamHandler(BaseHandler): auth_user, room_ids, pagin_config, timeout ) + time_now = self.clock.time_msec() + chunks = [ - self.hs.serialize_event(e, as_client_event) for e in events + serialize_event(e, time_now, as_client_event) for e in events ] chunk = { diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6a1104a890..9c3271fe88 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig +from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator from synapse.util.logcontext import PreserveLoggingContext from synapse.types import UserID @@ -100,9 +101,11 @@ class MessageHandler(BaseHandler): "room_key", next_key ) + time_now = self.clock.time_msec() + chunk = { "chunk": [ - self.hs.serialize_event(e, as_client_event) for e in events + serialize_event(e, time_now, as_client_event) for e in events ], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), @@ -211,7 +214,8 @@ class MessageHandler(BaseHandler): # TODO: This is duplicating logic from snapshot_all_rooms current_state = yield self.state_handler.get_current_state(room_id) - defer.returnValue([self.hs.serialize_event(c) for c in current_state]) + now = self.clock.time_msec() + defer.returnValue([serialize_event(c, now) for c in current_state]) @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, @@ -283,10 +287,11 @@ class MessageHandler(BaseHandler): start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) + time_now = self.clock.time_msec() d["messages"] = { "chunk": [ - self.hs.serialize_event(m, as_client_event) + serialize_event(m, time_now, as_client_event) for m in messages ], "start": start_token.to_string(), @@ -297,7 +302,8 @@ class MessageHandler(BaseHandler): event.room_id ) d["state"] = [ - self.hs.serialize_event(c) for c in current_state + serialize_event(c, time_now, as_client_event) + for c in current_state ] except: logger.exception("Failed to get snapshot") @@ -320,8 +326,9 @@ class MessageHandler(BaseHandler): auth_user = UserID.from_string(user_id) # TODO: These concurrently + time_now = self.clock.time_msec() state_tuples = yield self.state_handler.get_current_state(room_id) - state = [self.hs.serialize_event(x) for x in state_tuples] + state = [serialize_event(x, time_now) for x in state_tuples] member_event = (yield self.store.get_room_member( user_id=user_id, @@ -360,11 +367,13 @@ class MessageHandler(BaseHandler): "Failed to get member presence of %r", m.user_id ) + time_now = self.clock.time_msec() + defer.returnValue({ "membership": member_event.membership, "room_id": room_id, "messages": { - "chunk": [self.hs.serialize_event(m) for m in messages], + "chunk": [serialize_event(m, time_now) for m in messages], "start": start_token.to_string(), "end": end_token.to_string(), }, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index edb96cec83..23821d321f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -16,12 +16,14 @@ """Contains functions for performing events on rooms.""" from twisted.internet import defer +from ._base import BaseHandler + from synapse.types import UserID, RoomAlias, RoomID from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import StoreError, SynapseError from synapse.util import stringutils from synapse.util.async import run_on_reactor -from ._base import BaseHandler +from synapse.events.utils import serialize_event import logging @@ -293,8 +295,9 @@ class RoomMemberHandler(BaseHandler): yield self.auth.check_joined_room(room_id, user_id) member_list = yield self.store.get_room_members(room_id=room_id) + time_now = self.clock.time_msec() event_list = [ - self.hs.serialize_event(entry) + serialize_event(entry, time_now) for entry in member_list ] chunk_data = { diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index c69de56863..a0d051227b 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig from .base import ClientV1RestServlet, client_path_pattern +from synapse.events.utils import serialize_event import logging @@ -64,14 +65,19 @@ class EventStreamRestServlet(ClientV1RestServlet): class EventRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/events/(?P[^/]*)$") + def __init__(self, hs): + super(EventRestServlet, self).__init__(hs) + self.clock = hs.get_clock() + @defer.inlineCallbacks def on_GET(self, request, event_id): auth_user = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) + time_now = self.clock.time_msec() if event: - defer.returnValue((200, self.hs.serialize_event(event))) + defer.returnValue((200, serialize_event(event, time_now))) else: defer.returnValue((404, "Event not found.")) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f06e3ddb98..58b09b6fc1 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -21,6 +21,7 @@ from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID, RoomAlias +from synapse.events.utils import serialize_event import json import logging @@ -363,6 +364,10 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): class RoomTriggerBackfill(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/backfill$") + def __init__(self, hs): + super(RoomTriggerBackfill, self).__init__(hs) + self.clock = hs.get_clock() + @defer.inlineCallbacks def on_GET(self, request, room_id): remote_server = urllib.unquote( @@ -374,7 +379,9 @@ class RoomTriggerBackfill(ClientV1RestServlet): handler = self.handlers.federation_handler events = yield handler.backfill(remote_server, room_id, limit) - res = [self.hs.serialize_event(event) for event in events] + time_now = self.clock.time_msec() + + res = [serialize_event(event, time_now) for event in events] defer.returnValue((200, res)) diff --git a/synapse/server.py b/synapse/server.py index 92ed2c5e32..c478f812e6 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -124,9 +124,6 @@ class BaseHomeServer(object): setattr(BaseHomeServer, "get_%s" % (depname), _get) - def serialize_event(self, e, as_client_event=True): - return serialize_event(self, e, as_client_event) - def get_ip_from_request(self, request): # May be an X-Forwarding-For header depending on config ip_addr = request.getClientIP() -- cgit 1.4.1 From e5725eb3b950d7580672ed98b1366a175e41e916 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 16:16:50 +0000 Subject: Remove unused import from server.py --- synapse/server.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/server.py b/synapse/server.py index c478f812e6..f09d5d581e 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -20,7 +20,6 @@ # Imports required for the default HomeServer() implementation from synapse.federation import initialize_http_replication -from synapse.events.utils import serialize_event from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers -- cgit 1.4.1 From 69a75b7ebebb393c1ce84ff949f3480a6af0a782 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 26 Jan 2015 16:52:47 +0000 Subject: Add brackets to make get room name / alias work --- synapse/storage/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 8f56d90d95..2534d109fd 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -402,8 +402,8 @@ class DataStore(RoomMemberStore, RoomStore, "redacted": del_sql, } - sql += " AND (s.type = 'm.room.name' AND s.state_key = '')" - sql += " OR s.type = 'm.room.aliases'" + sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" + sql += " OR s.type = 'm.room.aliases')" args = (room_id,) results = yield self._execute_and_decode(sql, *args) -- cgit 1.4.1 From b481889117ceb8b31dd2e491bca53b914d306312 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 26 Jan 2015 17:27:28 +0000 Subject: Support membership events and more camelcase/underscores --- synapse/push/__init__.py | 4 ++++ synapse/push/httppusher.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 0c51d2dd8f..b6d01a82a0 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -65,6 +65,10 @@ class Pusher(object): # let's assume you probably know about messages you sent yourself defer.returnValue(['dont_notify']) + if ev['type'] == 'm.room.member': + if ev['state_key'] != self.user_name: + defer.returnValue(['dont_notify']) + rules = yield self.store.get_push_rules_for_user_name(self.user_name) for r in rules: diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 25db1dded5..22532fcc6a 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -65,8 +65,6 @@ class HttpPusher(Pusher): d = { 'notification': { - 'transition': 'new', - # everything is new for now: we don't have read receipts 'id': event['event_id'], 'type': event['type'], 'from': event['user_id'], @@ -89,11 +87,13 @@ class HttpPusher(Pusher): ] } } + if event['type'] == 'm.room.member': + d['notification']['membership'] = event['content']['membership'] if len(ctx['aliases']): - d['notification']['roomAlias'] = ctx['aliases'][0] + d['notification']['room_alias'] = ctx['aliases'][0] if 'name' in ctx: - d['notification']['roomName'] = ctx['name'] + d['notification']['room_name'] = ctx['name'] defer.returnValue(d) -- cgit 1.4.1 From 436513068de73ab47d9ba9a32046420be3d86588 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 18:53:31 +0000 Subject: Start implementing the non-incremental sync portion of the v2 /sync API --- synapse/events/utils.py | 6 +- synapse/handlers/__init__.py | 2 + synapse/handlers/sync.py | 87 ++++++++++++++++++++--------- synapse/rest/client/v2_alpha/sync.py | 105 +++++++++++++++++++++++++---------- 4 files changed, 146 insertions(+), 54 deletions(-) (limited to 'synapse') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index e391aca4cc..b7f1ad4b40 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,7 +89,7 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(e, time_now_ms, client_event=True): +def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e @@ -138,4 +138,8 @@ def serialize_event(e, time_now_ms, client_event=True): d.pop("unsigned", None) d.pop("origin", None) + if strip_ids: + d.pop("room_id", None) + d.pop("event_id", None) + return d diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index fe071a4bc2..a32eab9316 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -26,6 +26,7 @@ from .presence import PresenceHandler from .directory import DirectoryHandler from .typing import TypingNotificationHandler from .admin import AdminHandler +from .sync import SyncHandler class Handlers(object): @@ -51,3 +52,4 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) + self.sync_handler = SyncHandler(hs) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ec20ea4890..bbabaf3df1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1,26 +1,49 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import BaseHandler + +from synapse.streams.config import PaginationConfig +from synapse.api.constants import Membership + +from twisted.internet import defer + import collections +import logging + +logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", "device", - "since", "limit", "gap", - "sort" - "backfill" + "sort", + "backfill", "filter", -) +]) RoomSyncResult = collections.namedtuple("RoomSyncResult", [ "room_id", "limited", "published", - "prev_batch", - "events", + "events", # dict of event "state", - "event_map", + "prev_batch", ]) @@ -41,10 +64,11 @@ class SyncHandler(BaseHandler): def __init__(self, hs): super(SyncHandler, self).__init__(hs) self.event_sources = hs.get_event_sources() + self.clock = hs.get_clock() def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): if timeout == 0: - return self.current_sync_for_user(sync_config, since) + return self.current_sync_for_user(sync_config, since_token) else: def current_sync_callback(since_token): return self.current_sync_for_user( @@ -53,58 +77,71 @@ class SyncHandler(BaseHandler): return self.notifier.wait_for_events( sync_config.filter, since_token, current_sync_callback ) - defer.returnValue(result) def current_sync_for_user(self, sync_config, since_token=None): if since_token is None: - return self.inital_sync(sync_config) + return self.initial_sync(sync_config) else: return self.incremental_sync(sync_config) @defer.inlineCallbacks def initial_sync(self, sync_config): + if sync_config.sort == "timeline,desc": + # TODO(mjark): Handle going through events in reverse order?. + # What does "most recent events" mean when applying the limits mean + # in this case? + raise NotImplementedError() + now_token = yield self.event_sources.get_current_token() presence_stream = self.event_sources.sources["presence"] - # TODO (markjh): This looks wrong, shouldn't we be getting the presence + # TODO (mjark): This looks wrong, shouldn't we be getting the presence # UP to the present rather than after the present? pagination_config = PaginationConfig(from_token=now_token) presence, _ = yield presence_stream.get_pagination_rows( - user, pagination_config.get_source_config("presence"), None + user=sync_config.user, + pagination_config=pagination_config.get_source_config("presence"), + key=None ) room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=user_id, + user_id=sync_config.user.to_string(), membership_list=[Membership.INVITE, Membership.JOIN] ) - # TODO (markjh): Does public mean "published"? + # TODO (mjark): Does public mean "published"? published_rooms = yield self.store.get_rooms(is_public=True) - published_room_ids = set(r["room_id"] for r in public_rooms) + published_room_ids = set(r["room_id"] for r in published_rooms) + rooms = [] for event in room_list: - - messages, token = yield self.store.get_recent_events_for_room( + #TODO (mjark): Apply the event filter in sync_config. + recent_events, token = yield self.store.get_recent_events_for_room( event.room_id, limit=sync_config.limit, end_token=now_token.room_key, ) prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - current_state = yield self.state_handler.get_current_state( + current_state_events = yield self.state_handler.get_current_state( event.room_id ) rooms.append(RoomSyncResult( room_id=event.room_id, published=event.room_id in published_room_ids, + events=recent_events, + prev_batch=prev_batch_token, + state=current_state_events, + limited=True, + )) - - + defer.returnValue(SyncResult( + public_user_data=presence, + private_user_data=[], + rooms=rooms, + next_batch=now_token, + )) @defer.inlineCallbacks def incremental_sync(self, sync_config): - - - - - + pass diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 39bb5ec8e9..cc667ebafc 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,9 @@ from twisted.internet import defer from synapse.http.servlet import RestServlet +from synapse.handlers.sync import SyncConfig +from synapse.types import StreamToken +from synapse.events.utils import serialize_event from ._base import client_v2_pattern import logging @@ -73,14 +76,15 @@ class SyncRestServlet(RestServlet): def __init__(self, hs): super(SyncRestServlet, self).__init__() self.auth = hs.get_auth() - #self.sync_handler = hs.get_handlers().sync_hanlder + self.sync_handler = hs.get_handlers().sync_handler + self.clock = hs.get_clock() @defer.inlineCallbacks def on_GET(self, request): user = yield self.auth.get_user_by_req(request) timeout = self.parse_integer(request, "timeout", default=0) - limit = self.parse_integer(request, "limit", default=None) + limit = self.parse_integer(request, "limit", required=True) gap = self.parse_boolean(request, "gap", default=True) sort = self.parse_string( request, "sort", default="timeline,asc", @@ -91,7 +95,7 @@ class SyncRestServlet(RestServlet): request, "set_presence", default="online", allowed_values=self.ALLOWED_PRESENCE ) - backfill = self.parse_boolean(request, "backfill", default=True) + backfill = self.parse_boolean(request, "backfill", default=False) filter_id = self.parse_string(request, "filter", default=None) logger.info( @@ -108,36 +112,81 @@ class SyncRestServlet(RestServlet): # if filter.matches(event): # # stuff - # if timeout != 0: - # register for updates from the event stream - - #rooms = [] - - if gap: - pass - # now_stream_token = get_current_stream_token - # for room_id in get_rooms_for_user(user, filter=filter): - # state, events, start, end, limited, published = updates_for_room( - # from=since, to=now_stream_token, limit=limit, - # anchor_to_start=False - # ) - # rooms[room_id] = (state, events, start, limited, published) - # next_stream_token = now. + sync_config = SyncConfig( + user=user, + device="TODO", # TODO(mjark) Get the device_id from access_token + gap=gap, + limit=limit, + sort=sort, + backfill=backfill, + filter="TODO", # TODO(mjark) Add the filter to the config. + ) + + if since is not None: + since_token = StreamToken.from_string(since) else: - pass - # now_stream_token = get_current_stream_token - # for room_id in get_rooms_for_user(user, filter=filter) - # state, events, start, end, limited, published = updates_for_room( - # from=since, to=now_stream_token, limit=limit, - # anchor_to_start=False - # ) - # next_stream_token = min(next_stream_token, end) + since_token = None + sync_result = yield self.sync_handler.wait_for_sync_for_user( + sync_config, since_token=since_token, timeout=timeout + ) - response_content = {} + time_now = self.clock.time_msec() + + response_content = { + "public_user_data": self.encode_events( + sync_result.public_user_data, filter, time_now + ), + "private_user_data": self.encode_events( + sync_result.private_user_data, filter, time_now + ), + "rooms": self.encode_rooms(sync_result.rooms, filter, time_now), + "next_batch": sync_result.next_batch.to_string(), + } defer.returnValue((200, response_content)) + def encode_events(self, events, filter, time_now): + return [self.encode_event(event, filter, time_now) for event in events] + + @staticmethod + def encode_event(event, filter, time_now): + # TODO(mjark): Respect formatting requirements in the filter. + return serialize_event(event, time_now) + + def encode_rooms(self, rooms, filter, time_now): + return [self.encode_room(room, filter, time_now) for room in rooms] + + @staticmethod + def encode_room(room, filter, time_now): + event_map = {} + state_event_ids = [] + recent_event_ids = [] + for event in room.state: + # TODO(mjark): Respect formatting requirements in the filter. + event_map[event.event_id] = serialize_event( + event, time_now, strip_ids=True + ) + state_event_ids.append(event.event_id) + + for event in room.events: + # TODO(mjark): Respect formatting requirements in the filter. + event_map[event.event_id] = serialize_event( + event, time_now, strip_ids=True + ) + recent_event_ids.append(event.event_id) + return { + "room_id": room.room_id, + "event_map": event_map, + "events": { + "batch": recent_event_ids, + "prev_batch": room.prev_batch.to_string(), + }, + "state": state_event_ids, + "limited": room.limited, + "published": room.published, + } + def register_servlets(hs, http_server): SyncRestServlet(hs).register(http_server) -- cgit 1.4.1 From fa8e6ff90074708ec19fcdcdfd79c1dce03cb128 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 14:01:51 +0000 Subject: Add stub application services REST API. --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 7 ++++- synapse/rest/appservice/__init__.py | 14 ++++++++++ synapse/rest/appservice/v1/__init__.py | 29 +++++++++++++++++++++ synapse/rest/appservice/v1/base.py | 47 ++++++++++++++++++++++++++++++++++ synapse/rest/appservice/v1/register.py | 37 ++++++++++++++++++++++++++ synapse/server.py | 1 + 7 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/appservice/__init__.py create mode 100644 synapse/rest/appservice/v1/__init__.py create mode 100644 synapse/rest/appservice/v1/base.py create mode 100644 synapse/rest/appservice/v1/register.py (limited to 'synapse') diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 693c0efda6..9485719332 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -22,3 +22,4 @@ WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_PREFIX = "/_matrix/key/v1" MEDIA_PREFIX = "/_matrix/media/v1" +APP_SERVICE_PREFIX = "/_matrix/appservice/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 40d28dcbdc..f7e24d0cc6 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,13 +26,14 @@ from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect +from synapse.rest.appservice.v1 import AppServiceRestResource from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, APP_SERVICE_PREFIX ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory @@ -69,6 +70,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_federation(self): return JsonResource() + def build_resource_for_app_services(self): + return AppServiceRestResource(self) + def build_resource_for_web_client(self): syweb_path = os.path.dirname(syweb.__file__) webclient_path = os.path.join(syweb_path, "webclient") @@ -114,6 +118,7 @@ class SynapseHomeServer(HomeServer): (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), (MEDIA_PREFIX, self.get_resource_for_media_repository()), + (APP_SERVICE_PREFIX, self.get_resource_for_app_services()), ] if web_client: logger.info("Adding the web client.") diff --git a/synapse/rest/appservice/__init__.py b/synapse/rest/appservice/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/synapse/rest/appservice/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py new file mode 100644 index 0000000000..bf243b6180 --- /dev/null +++ b/synapse/rest/appservice/v1/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from . import register + +from synapse.http.server import JsonResource + + +class AppServiceRestResource(JsonResource): + """A resource for version 1 of the matrix application service API.""" + + def __init__(self, hs): + JsonResource.__init__(self) + self.register_servlets(self, hs) + + @staticmethod + def register_servlets(appservice_resource, hs): + register.register_servlets(hs, appservice_resource) \ No newline at end of file diff --git a/synapse/rest/appservice/v1/base.py b/synapse/rest/appservice/v1/base.py new file mode 100644 index 0000000000..46c9a444c0 --- /dev/null +++ b/synapse/rest/appservice/v1/base.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.http.servlet import RestServlet +from synapse.api.urls import APP_SERVICE_PREFIX +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def as_path_pattern(path_regex): + """Creates a regex compiled appservice path with the correct path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + APP_SERVICE_PREFIX + path_regex) + + +class AppServiceRestServlet(RestServlet): + """A base Synapse REST Servlet for the application services version 1 API. + """ + + def __init__(self, hs): + self.hs = hs \ No newline at end of file diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py new file mode 100644 index 0000000000..17b6dc5664 --- /dev/null +++ b/synapse/rest/appservice/v1/register.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains REST servlets to do with registration: /register""" + +from base import AppServiceRestServlet, as_path_pattern +from synapse.api.errors import CodeMessageException + +import logging + +logger = logging.getLogger(__name__) + + +class RegisterRestServlet(AppServiceRestServlet): + """Handles AS registration with the home server. + """ + + PATTERN = as_path_pattern("/register$") + + def on_POST(self, request): + raise CodeMessageException(500, "Not implemented") + + +def register_servlets(hs, http_server): + RegisterRestServlet(hs).register(http_server) \ No newline at end of file diff --git a/synapse/server.py b/synapse/server.py index f09d5d581e..891c5aa13d 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -75,6 +75,7 @@ class BaseHomeServer(object): 'resource_for_content_repo', 'resource_for_server_key', 'resource_for_media_repository', + 'resource_for_app_services', 'event_sources', 'ratelimiter', 'keyring', -- cgit 1.4.1 From 05c7cba73a050f19cc52129b65b0183eaa832a42 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 14:28:56 +0000 Subject: Initial trivial implementation of an actual 'Filtering' object; move storage of user filters into there --- synapse/api/filtering.py | 41 ++++++++++++++++++++++++++++++++++ synapse/rest/client/v2_alpha/filter.py | 25 ++++++++++----------- synapse/server.py | 5 +++++ 3 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 synapse/api/filtering.py (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py new file mode 100644 index 0000000000..922c40004c --- /dev/null +++ b/synapse/api/filtering.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# TODO(paul) +_filters_for_user = {} + + +class Filtering(object): + + def __init__(self, hs): + super(Filtering, self).__init__() + self.hs = hs + + def get_user_filter(self, user_localpart, filter_id): + filters = _filters_for_user.get(user_localpart, None) + + if not filters or filter_id >= len(filters): + raise KeyError() + + return filters[filter_id] + + def add_user_filter(self, user_localpart, definition): + filters = _filters_for_user.setdefault(user_localpart, []) + + filter_id = len(filters) + filters.append(definition) + + return filter_id diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index a9a180ec04..585c8e02e8 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -28,10 +28,6 @@ import logging logger = logging.getLogger(__name__) -# TODO(paul) -_filters_for_user = {} - - class GetFilterRestServlet(RestServlet): PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter/(?P[^/]*)") @@ -39,6 +35,7 @@ class GetFilterRestServlet(RestServlet): super(GetFilterRestServlet, self).__init__() self.hs = hs self.auth = hs.get_auth() + self.filtering = hs.get_filtering() @defer.inlineCallbacks def on_GET(self, request, user_id, filter_id): @@ -56,13 +53,14 @@ class GetFilterRestServlet(RestServlet): except: raise SynapseError(400, "Invalid filter_id") - filters = _filters_for_user.get(target_user.localpart, None) - - if not filters or filter_id >= len(filters): + try: + defer.returnValue((200, self.filtering.get_user_filter( + user_localpart=target_user.localpart, + filter_id=filter_id, + ))) + except KeyError: raise SynapseError(400, "No such filter") - defer.returnValue((200, filters[filter_id])) - class CreateFilterRestServlet(RestServlet): PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter") @@ -71,6 +69,7 @@ class CreateFilterRestServlet(RestServlet): super(CreateFilterRestServlet, self).__init__() self.hs = hs self.auth = hs.get_auth() + self.filtering = hs.get_filtering() @defer.inlineCallbacks def on_POST(self, request, user_id): @@ -90,10 +89,10 @@ class CreateFilterRestServlet(RestServlet): except: raise SynapseError(400, "Invalid filter definition") - filters = _filters_for_user.setdefault(target_user.localpart, []) - - filter_id = len(filters) - filters.append(content) + filter_id = self.filtering.add_user_filter( + user_localpart=target_user.localpart, + definition=content, + ) defer.returnValue((200, {"filter_id": str(filter_id)})) diff --git a/synapse/server.py b/synapse/server.py index f09d5d581e..9b42079e05 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -32,6 +32,7 @@ from synapse.streams.events import EventSources from synapse.api.ratelimiting import Ratelimiter from synapse.crypto.keyring import Keyring from synapse.events.builder import EventBuilderFactory +from synapse.api.filtering import Filtering class BaseHomeServer(object): @@ -79,6 +80,7 @@ class BaseHomeServer(object): 'ratelimiter', 'keyring', 'event_builder_factory', + 'filtering', ] def __init__(self, hostname, **kwargs): @@ -197,3 +199,6 @@ class HomeServer(BaseHomeServer): clock=self.get_clock(), hostname=self.hostname, ) + + def build_filtering(self): + return Filtering(self) -- cgit 1.4.1 From 6efdc11cc811fedd5c989dc06df64fd334b4b033 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 15:03:19 +0000 Subject: Parse /register and /unregister request JSON. --- synapse/rest/appservice/v1/register.py | 80 +++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 17b6dc5664..779447ac6a 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -16,8 +16,9 @@ """This module contains REST servlets to do with registration: /register""" from base import AppServiceRestServlet, as_path_pattern -from synapse.api.errors import CodeMessageException +from synapse.api.errors import CodeMessageException, SynapseError +import json import logging logger = logging.getLogger(__name__) @@ -30,8 +31,83 @@ class RegisterRestServlet(AppServiceRestServlet): PATTERN = as_path_pattern("/register$") def on_POST(self, request): + params = _parse_json(request) + + # sanity check required params + try: + as_token = params["as_token"] + as_url = params["url"] + if (not isinstance(as_token, basestring) or + not isinstance(as_url, basestring)): + raise ValueError + except (KeyError, ValueError): + raise SynapseError( + 400, "Missed required keys: as_token(str) / url(str)." + ) + + namespaces = { + "users": [], + "rooms": [], + "aliases": [] + } + + if "namespaces" in params: + self._parse_namespace(namespaces, params["namespaces"], "users") + self._parse_namespace(namespaces, params["namespaces"], "rooms") + self._parse_namespace(namespaces, params["namespaces"], "aliases") + + # TODO: pass to the appservice handler + + raise CodeMessageException(500, "Not implemented.") + + def _parse_namespace(self, target_ns, origin_ns, ns): + if ns not in target_ns or ns not in origin_ns: + return # nothing to parse / map through to. + + possible_regex_list = origin_ns[ns] + if not type(possible_regex_list) == list: + raise SynapseError(400, "Namespace %s isn't an array." % ns) + + for regex in possible_regex_list: + if not isinstance(regex, basestring): + raise SynapseError( + 400, "Regex '%s' isn't a string in namespace %s" % + (regex, ns) + ) + + target_ns[ns] = origin_ns[ns] + + +class UnregisterRestServlet(AppServiceRestServlet): + """Handles AS registration with the home server. + """ + + PATTERN = as_path_pattern("/unregister$") + + def on_POST(self, request): + params = _parse_json(request) + try: + as_token = params["as_token"] + if not isinstance(as_token, basestring): + raise ValueError + except (KeyError, ValueError): + raise SynapseError(400, "Missing required key: as_token(str)") + + # TODO: pass to the appservice handler + raise CodeMessageException(500, "Not implemented") +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.") + return content + except ValueError: + raise SynapseError(400, "Content not JSON.") + + def register_servlets(hs, http_server): - RegisterRestServlet(hs).register(http_server) \ No newline at end of file + RegisterRestServlet(hs).register(http_server) + UnregisterRestServlet(hs).register(http_server) -- cgit 1.4.1 From 51449e06654c4af7a645124dc64e1f0cc1678b24 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 15:50:28 +0000 Subject: Add appservice handler and store. Glue together rest > handler > store. --- synapse/handlers/__init__.py | 2 ++ synapse/handlers/appservice.py | 49 ++++++++++++++++++++++++++++++++++ synapse/rest/appservice/v1/base.py | 3 ++- synapse/rest/appservice/v1/register.py | 8 ++++-- synapse/storage/__init__.py | 5 ++-- synapse/storage/appservice.py | 45 +++++++++++++++++++++++++++++++ 6 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 synapse/handlers/appservice.py create mode 100644 synapse/storage/appservice.py (limited to 'synapse') diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index fe071a4bc2..96a9b143ca 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -26,6 +26,7 @@ from .presence import PresenceHandler from .directory import DirectoryHandler from .typing import TypingNotificationHandler from .admin import AdminHandler +from .appservice import ApplicationServicesHandler class Handlers(object): @@ -51,3 +52,4 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) + self.appservice_handler = ApplicationServicesHandler(hs) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py new file mode 100644 index 0000000000..55a653476f --- /dev/null +++ b/synapse/handlers/appservice.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import BaseHandler +from synapse.api.errors import StoreError, SynapseError + +import logging + + +logger = logging.getLogger(__name__) + + +class ApplicationServicesHandler(BaseHandler): + + def __init__(self, hs): + super(ApplicationServicesHandler, self).__init__(hs) + + @defer.inlineCallbacks + def register(self, base_url, token, namespaces): + # check the token is recognised + try: + app_service = yield self.store.get_app_service(token) + if not app_service: + raise StoreError + except StoreError: + raise SynapseError( + 403, "Unrecognised application services token. " + "Consult the home server admin." + ) + + # update AS entry with base URL + + # store namespaces for this AS + + defer.returnValue("not_implemented_yet") diff --git a/synapse/rest/appservice/v1/base.py b/synapse/rest/appservice/v1/base.py index 46c9a444c0..65d5bcf9be 100644 --- a/synapse/rest/appservice/v1/base.py +++ b/synapse/rest/appservice/v1/base.py @@ -44,4 +44,5 @@ class AppServiceRestServlet(RestServlet): """ def __init__(self, hs): - self.hs = hs \ No newline at end of file + self.hs = hs + self.handler = hs.get_handlers().appservice_handler diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 779447ac6a..142f09a638 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -14,6 +14,7 @@ # limitations under the License. """This module contains REST servlets to do with registration: /register""" +from twisted.internet import defer from base import AppServiceRestServlet, as_path_pattern from synapse.api.errors import CodeMessageException, SynapseError @@ -30,6 +31,7 @@ class RegisterRestServlet(AppServiceRestServlet): PATTERN = as_path_pattern("/register$") + @defer.inlineCallbacks def on_POST(self, request): params = _parse_json(request) @@ -56,9 +58,11 @@ class RegisterRestServlet(AppServiceRestServlet): self._parse_namespace(namespaces, params["namespaces"], "rooms") self._parse_namespace(namespaces, params["namespaces"], "aliases") - # TODO: pass to the appservice handler + hs_token = yield self.handler.register(as_url, as_token, namespaces) - raise CodeMessageException(500, "Not implemented.") + defer.returnValue({ + "hs_token": hs_token + }) def _parse_namespace(self, target_ns, origin_ns, ns): if ns not in target_ns or ns not in origin_ns: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4beb951b9f..9431c1a32d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.api.constants import EventTypes +from .appservice import ApplicationServiceStore from .directory import DirectoryStore from .feedback import FeedbackStore from .presence import PresenceStore @@ -80,8 +81,8 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, - EventFederationStore, - MediaRepositoryStore, + EventFederationStore, MediaRepositoryStore, + ApplicationServiceStore ): def __init__(self, hs): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py new file mode 100644 index 0000000000..99f58b4c62 --- /dev/null +++ b/synapse/storage/appservice.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import StoreError + +from ._base import SQLBaseStore + + +class ApplicationServiceStore(SQLBaseStore): + + def __init__(self, hs): + super(ApplicationServiceStore, self).__init__(hs) + + self.clock = hs.get_clock() + + @defer.inlineCallbacks + def get_app_service(self, as_token): + """Get the application service with the given token. + + Args: + token (str): The application service token. + Raises: + StoreError if there was a problem retrieving this. + """ + row = self._simple_select_one( + "application_services", {"token": as_token}, + ["url", "token"] + ) + if not row: + raise StoreError(400, "Bad application services token supplied.") + defer.returnValue(row) -- cgit 1.4.1 From 1d779691248be2fcdd43e97352d13c2cef239c10 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 15:58:27 +0000 Subject: Unbreak bad presence merge - don't add these blocks together with an and: they're different things. --- synapse/handlers/events.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 75fb941008..f1a3e4a4ad 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -51,19 +51,21 @@ class EventStreamHandler(BaseHandler): auth_user = self.hs.parse_userid(auth_user_id) try: - if affect_presence and auth_user not in self._streams_per_user: - self._streams_per_user[auth_user] = 0 - if auth_user in self._stop_timer_per_user: - try: - self.clock.cancel_call_later( - self._stop_timer_per_user.pop(auth_user) + if affect_presence: + if auth_user not in self._streams_per_user: + self._streams_per_user[auth_user] = 0 + if auth_user in self._stop_timer_per_user: + try: + print "cancel",auth_user + self.clock.cancel_call_later( + self._stop_timer_per_user.pop(auth_user) + ) + except: + logger.exception("Failed to cancel event timer") + else: + yield self.distributor.fire( + "started_user_eventstream", auth_user ) - except: - logger.exception("Failed to cancel event timer") - else: - yield self.distributor.fire( - "started_user_eventstream", auth_user - ) self._streams_per_user[auth_user] += 1 if pagin_config.from_token is None: -- cgit 1.4.1 From eba89f093f0376b8499953b53bbd85f9b12b7852 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:00:07 +0000 Subject: Need a defer.inlineCallbacks here as we yield in it: otherwise nothing in the cb gets executed. --- synapse/handlers/events.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index f1a3e4a4ad..851eebf600 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -99,6 +99,7 @@ class EventStreamHandler(BaseHandler): # 10 seconds of grace to allow the client to reconnect again # before we think they're gone + @defer.inlineCallbacks def _later(): logger.debug( "_later stopped_user_eventstream %s", auth_user -- cgit 1.4.1 From 5eacaeb4a73aa4c4ff450d7b23acabe79124171f Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:05:23 +0000 Subject: or of course we could just return the deferred --- synapse/handlers/events.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 851eebf600..48de3630e3 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -99,7 +99,6 @@ class EventStreamHandler(BaseHandler): # 10 seconds of grace to allow the client to reconnect again # before we think they're gone - @defer.inlineCallbacks def _later(): logger.debug( "_later stopped_user_eventstream %s", auth_user @@ -107,7 +106,7 @@ class EventStreamHandler(BaseHandler): self._stop_timer_per_user.pop(auth_user, None) - yield self.distributor.fire( + return self.distributor.fire( "stopped_user_eventstream", auth_user ) -- cgit 1.4.1 From f7c4daa8f94e5968277ba438f6d3da1f0f27ba4c Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:08:47 +0000 Subject: Oops, remove debugging --- synapse/handlers/events.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 48de3630e3..52ef07eaae 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -56,7 +56,6 @@ class EventStreamHandler(BaseHandler): self._streams_per_user[auth_user] = 0 if auth_user in self._stop_timer_per_user: try: - print "cancel",auth_user self.clock.cancel_call_later( self._stop_timer_per_user.pop(auth_user) ) -- cgit 1.4.1 From 059651efa19a88eb0823bce1d5beff2d95cb01c2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 16:17:56 +0000 Subject: Have the Filtering API return Deferreds, so we can do the Datastore implementation nicely --- synapse/api/filtering.py | 16 ++++++++++++++-- synapse/rest/client/v2_alpha/filter.py | 8 +++++--- tests/api/test_filtering.py | 5 +++-- 3 files changed, 22 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 922c40004c..014e2e1fc9 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + # TODO(paul) _filters_for_user = {} @@ -24,18 +26,28 @@ class Filtering(object): super(Filtering, self).__init__() self.hs = hs + @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): filters = _filters_for_user.get(user_localpart, None) if not filters or filter_id >= len(filters): raise KeyError() - return filters[filter_id] + # trivial yield to make it a generator so d.iC works + yield + defer.returnValue(filters[filter_id]) + @defer.inlineCallbacks def add_user_filter(self, user_localpart, definition): filters = _filters_for_user.setdefault(user_localpart, []) filter_id = len(filters) filters.append(definition) - return filter_id + # trivial yield, see above + yield + defer.returnValue(filter_id) + + # TODO(paul): surely we should probably add a delete_user_filter or + # replace_user_filter at some point? There's no REST API specified for + # them however diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 585c8e02e8..09e44e8ae0 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -54,10 +54,12 @@ class GetFilterRestServlet(RestServlet): raise SynapseError(400, "Invalid filter_id") try: - defer.returnValue((200, self.filtering.get_user_filter( + filter = yield self.filtering.get_user_filter( user_localpart=target_user.localpart, filter_id=filter_id, - ))) + ) + + defer.returnValue((200, filter)) except KeyError: raise SynapseError(400, "No such filter") @@ -89,7 +91,7 @@ class CreateFilterRestServlet(RestServlet): except: raise SynapseError(400, "Invalid filter definition") - filter_id = self.filtering.add_user_filter( + filter_id = yield self.filtering.add_user_filter( user_localpart=target_user.localpart, definition=content, ) diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index c6c5317696..fecadd1056 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -53,14 +53,15 @@ class FilteringTestCase(unittest.TestCase): self.filtering = hs.get_filtering() + @defer.inlineCallbacks def test_filter(self): - filter_id = self.filtering.add_user_filter( + filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, definition={"type": ["m.*"]}, ) self.assertEquals(filter_id, 0) - filter = self.filtering.get_user_filter( + filter = yield self.filtering.get_user_filter( user_localpart=user_localpart, filter_id=filter_id, ) -- cgit 1.4.1 From 7331d34839dca468b9e396e1d2952b0bb32011bf Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 16:23:46 +0000 Subject: Add AS specific classes with docstrings. --- synapse/storage/appservice.py | 60 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 99f58b4c62..4c11191fe8 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -20,11 +20,69 @@ from synapse.api.errors import StoreError from ._base import SQLBaseStore +# XXX: This feels like it should belong in a "models" module, not storage. +class ApplicationService(object): + """Defines an application service. + + Provides methods to check if this service is "interested" in events. + """ + + def __init__(self, token, url=None, namespaces=None): + self.token = token + if url: + self.url = url + if namespaces: + self.namespaces = namespaces + + def is_interested(self, event): + """Check if this service is interested in this event. + + Args: + event(Event): The event to check. + Returns: + bool: True if this service would like to know about this event. + """ + # NB: This does not check room alias regex matches because that requires + # more context that an Event can provide. Room alias matches are checked + # in the ApplicationServiceHandler. + + # TODO check if event.room_id regex matches + # TODO check if event.user_id regex matches (or m.room.member state_key) + + return True + + +class ApplicationServiceCache(object): + """Caches ApplicationServices and provides utility functions on top. + + This class is designed to be invoked on incoming events in order to avoid + hammering the database every time to extract a list of application service + regexes. + """ + + def __init__(self): + self.services = [] + + def get_services_for_event(self, event): + """Retrieve a list of application services interested in this event. + + Args: + event(Event): The event to check. + Returns: + list: A list of services interested in this + event based on the service regex. + """ + interested_list = [ + s for s in self.services if s.is_event_claimed(event) + ] + return interested_list + + class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) - + self.cache = ApplicationServiceCache() self.clock = hs.get_clock() @defer.inlineCallbacks -- cgit 1.4.1 From a56008842b43089433768f569f35b2d14523ac39 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 16:24:22 +0000 Subject: Start implementing incremental initial sync --- synapse/events/utils.py | 1 + synapse/handlers/sync.py | 233 +++++++++++++++++++++++++++++++++++++++++----- synapse/storage/stream.py | 41 ++++++-- 3 files changed, 241 insertions(+), 34 deletions(-) (limited to 'synapse') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index b7f1ad4b40..42fb0371e5 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -137,6 +137,7 @@ def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): d.pop("depth", None) d.pop("unsigned", None) d.pop("origin", None) + d.pop("prev_state", None) if strip_ids: d.pop("room_id", None) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index bbabaf3df1..f8629a588f 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -37,14 +37,18 @@ SyncConfig = collections.namedtuple("SyncConfig", [ ]) -RoomSyncResult = collections.namedtuple("RoomSyncResult", [ +class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "room_id", "limited", "published", - "events", # dict of event + "events", "state", "prev_batch", -]) +])): + __slots__ = [] + + def __nonzero__(self): + return bool(self.events or self.state) class SyncResult(collections.namedtuple("SyncResult", [ @@ -56,7 +60,9 @@ class SyncResult(collections.namedtuple("SyncResult", [ __slots__ = [] def __nonzero__(self): - return self.private_user_data or self.public_user_data or self.rooms + return bool( + self.private_user_data or self.public_user_data or self.rooms + ) class SyncHandler(BaseHandler): @@ -67,7 +73,13 @@ class SyncHandler(BaseHandler): self.clock = hs.get_clock() def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): - if timeout == 0: + """Get the sync for a client if we have new data for it now. Otherwise + wait for new data to arrive on the server. If the timeout expires, then + return an empty sync result. + Returns: + A Deferred SyncResult. + """ + if timeout == 0 or since_token is None: return self.current_sync_for_user(sync_config, since_token) else: def current_sync_callback(since_token): @@ -79,13 +91,25 @@ class SyncHandler(BaseHandler): ) def current_sync_for_user(self, sync_config, since_token=None): + """Get the sync for client needed to match what the server has now. + Returns: + A Deferred SyncResult. + """ if since_token is None: return self.initial_sync(sync_config) else: - return self.incremental_sync(sync_config) + if sync_config.gap: + return self.incremental_sync_with_gap(sync_config, since_token) + else: + #TODO(mjark): Handle gapless sync + pass @defer.inlineCallbacks def initial_sync(self, sync_config): + """Get a sync for a client which is starting without any state + Returns: + A Deferred SyncResult. + """ if sync_config.sort == "timeline,desc": # TODO(mjark): Handle going through events in reverse order?. # What does "most recent events" mean when applying the limits mean @@ -114,25 +138,86 @@ class SyncHandler(BaseHandler): rooms = [] for event in room_list: - #TODO (mjark): Apply the event filter in sync_config. - recent_events, token = yield self.store.get_recent_events_for_room( - event.room_id, - limit=sync_config.limit, - end_token=now_token.room_key, - ) - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - current_state_events = yield self.state_handler.get_current_state( - event.room_id + room_sync = yield self.initial_sync_for_room( + event.room_id, sync_config, now_token, published_room_ids ) + rooms.append(room_sync) + + defer.returnValue(SyncResult( + public_user_data=presence, + private_user_data=[], + rooms=rooms, + next_batch=now_token, + )) + + @defer.inlineCallbacks + def intial_sync_for_room(self, room_id, sync_config, now_token, + published_room_ids): + """Sync a room for a client which is starting without any state + Returns: + A Deferred RoomSyncResult. + """ + recent_events, token = yield self.store.get_recent_events_for_room( + room_id, + limit=sync_config.limit, + end_token=now_token.room_key, + ) + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state_events = yield self.state_handler.get_current_state( + room_id + ) + + defer.returnValue(RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recent_events, + prev_batch=prev_batch_token, + state=current_state_events, + limited=True, + )) + + + @defer.inlineCallbacks + def incremental_sync_with_gap(self, sync_config, since_token): + """ Get the incremental delta needed to bring the client up to + date with the server. + Returns: + A Deferred SyncResult. + """ + if sync_config.sort == "timeline,desc": + # TODO(mjark): Handle going through events in reverse order?. + # What does "most recent events" mean when applying the limits mean + # in this case? + raise NotImplementedError() + + now_token = yield self.event_sources.get_current_token() + + presence_stream = self.event_sources.sources["presence"] + pagination_config = PaginationConfig( + from_token=since_token, to_token=now_token + ) + presence, _ = yield presence_stream.get_pagination_rows( + user=sync_config.user, + pagination_config=pagination_config.get_source_config("presence"), + key=None + ) + room_list = yield self.store.get_rooms_for_user_where_membership_is( + user_id=sync_config.user.to_string(), + membership_list=[Membership.INVITE, Membership.JOIN] + ) + + # TODO (mjark): Does public mean "published"? + published_rooms = yield self.store.get_rooms(is_public=True) + published_room_ids = set(r["room_id"] for r in published_rooms) - rooms.append(RoomSyncResult( - room_id=event.room_id, - published=event.room_id in published_room_ids, - events=recent_events, - prev_batch=prev_batch_token, - state=current_state_events, - limited=True, - )) + rooms = [] + for event in room_list: + room_sync = yield self.incremental_sync_with_gap_for_room( + event.room_id, sync_config, since_token, now_token, + published_room_ids + ) + if room_sync: + rooms.append(room_sync) defer.returnValue(SyncResult( public_user_data=presence, @@ -143,5 +228,103 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks - def incremental_sync(self, sync_config): - pass + def incremental_sync_with_gap_for_room(self, room_id, sync_config, + since_token, now_token, + published_room_ids): + """ Get the incremental delta needed to bring the client up to date for + the room. Gives the client the most recent events and the changes to + state. + Returns: + A Deferred RoomSyncResult + """ + # TODO(mjark): Check if they have joined the room between + # the previous sync and this one. + # TODO(mjark): Apply the event filter in sync_config + # TODO(mjark): Check for redactions we might have missed. + # TODO(mjark): Typing notifications. + recents, token = yield self.store.get_recent_events_for_room( + room_id, + limit=sync_config.limit + 1, + from_token=since_token.room_key, + end_token=now_token.room_key, + ) + + logging.debug("Recents %r", recents) + + if len(recents) > sync_config.limit: + limited = True + recents = recents[1:] + else: + limited = False + + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + + # TODO(mjark): This seems racy since this isn't being passed a + # token to indicate what point in the stream this is + current_state_events = yield self.state_handler.get_current_state( + room_id + ) + + state_at_previous_sync = yield self.get_state_at_previous_sync( + room_id, since_token=since_token + ) + + state_events_delta = yield self.compute_state_delta( + since_token=since_token, + previous_state=state_at_previous_sync, + current_state=current_state_events, + ) + + room_sync = RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recents, + prev_batch=prev_batch_token, + state=state_events_delta, + limited=limited, + ) + + logging.debug("Room sync: %r", room_sync) + + defer.returnValue(room_sync) + + @defer.inlineCallbacks + def get_state_at_previous_sync(self, room_id, since_token): + """ Get the room state at the previous sync the client made. + Returns: + A Deferred list of Events. + """ + last_events, token = yield self.store.get_recent_events_for_room( + room_id, end_token=since_token.room_key, limit=1, + ) + + if last_events: + last_event = last_events[0] + last_context = yield self.state_handler.compute_event_context( + last_event + ) + if last_event.is_state(): + state = [last_event] + last_context.current_state.values() + else: + state = last_context.current_state.values() + else: + state = () + defer.returnValue(state) + + + def compute_state_delta(self, since_token, previous_state, current_state): + """ Works out the differnce in state between the current state and the + state the client got when it last performed a sync. + Returns: + A list of events. + """ + # TODO(mjark) Check if the state events were received by the server + # after the previous sync, since we need to include those state + # updates even if they occured logically before the previous event. + # TODO(mjark) Check for new redactions in the state events. + previous_dict = {event.event_id:event for event in previous_state} + state_delta = [] + for event in current_state: + if event.event_id not in previous_dict: + state_delta.append(event) + return state_delta diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8ac2adab05..06aca1a4e5 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -265,17 +265,38 @@ class StreamStore(SQLBaseStore): return self.runInteraction("paginate_room_events", f) def get_recent_events_for_room(self, room_id, limit, end_token, - with_feedback=False): + with_feedback=False, from_token=None): # TODO (erikj): Handle compressed feedback - sql = ( - "SELECT stream_ordering, topological_ordering, event_id FROM events " - "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 " - "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " - ) + end_token = _StreamToken.parse_stream_token(end_token) - def f(txn): - txn.execute(sql, (room_id, end_token, limit,)) + if from_token is None: + sql = ( + "SELECT stream_ordering, topological_ordering, event_id" + " FROM events" + " WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0" + " ORDER BY topological_ordering DESC, stream_ordering DESC" + " LIMIT ?" + ) + else: + from_token = _StreamToken.parse_stream_token(from_token) + sql = ( + "SELECT stream_ordering, topological_ordering, event_id" + " FROM events" + " WHERE room_id = ? AND stream_ordering > ?" + " AND stream_ordering <= ? AND outlier = 0" + " ORDER BY topological_ordering DESC, stream_ordering DESC" + " LIMIT ?" + ) + + + def get_recent_events_for_room_txn(txn): + if from_token is None: + txn.execute(sql, (room_id, end_token.stream, limit,)) + else: + txn.execute(sql, ( + room_id, from_token.stream, end_token.stream, limit + )) rows = self.cursor_to_dict(txn) @@ -303,7 +324,9 @@ class StreamStore(SQLBaseStore): return events, token - return self.runInteraction("get_recent_events_for_room", f) + return self.runInteraction( + "get_recent_events_for_room", get_recent_events_for_room_txn + ) def get_room_events_max_id(self): return self.runInteraction( -- cgit 1.4.1 From 92171f9dd1ecac24aeae2f46729f3cbbbe94f91e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 16:53:59 +0000 Subject: Add stub methods, TODOs and docstrings for application services. --- synapse/handlers/appservice.py | 25 +++++++++++++++++--- synapse/storage/appservice.py | 52 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 55a653476f..25e1cece56 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -42,8 +42,27 @@ class ApplicationServicesHandler(BaseHandler): "Consult the home server admin." ) - # update AS entry with base URL - - # store namespaces for this AS + # store this AS defer.returnValue("not_implemented_yet") + + def unregister(self, token): + yield self.store.unregister_app_service(token) + + def notify_interested_services(self, event): + """Notifies (pushes) all application services interested in this event. + + Pushing is done asynchronously, so this method won't block for any + prolonged length of time. + + Args: + event(Event): The event to push out to interested services. + """ + # TODO: Gather interested services + # get_services_for_event(event) <-- room IDs and user IDs + # Get a list of room aliases. Check regex. + # TODO: If unknown user: poke User Query API. + # TODO: If unknown room alias: poke Room Alias Query API. + + # TODO: Fork off pushes to these services - XXX First cut, best effort + pass diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 4c11191fe8..fbad17cb9e 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -84,16 +84,60 @@ class ApplicationServiceStore(SQLBaseStore): super(ApplicationServiceStore, self).__init__(hs) self.cache = ApplicationServiceCache() self.clock = hs.get_clock() + self._populate_cache() + + def unregister_app_service(self, token): + """Unregisters this service. + + This removes all AS specific regex and the base URL. The token is the + only thing preserved for future registration attempts. + """ + # TODO: DELETE FROM application_services_regex WHERE id=this service + # TODO: SET url=NULL WHERE token=token + # TODO: Update cache + pass + + def update_app_service(self, service): + """Update an application service, clobbering what was previously there. + + Args: + service(ApplicationService): The updated service. + """ + # NB: There is no "insert" since we provide no public-facing API to + # allocate new ASes. It relies on the server admin inserting the AS + # token into the database manually. + + # TODO: UPDATE application_services, SET url WHERE token=service.token + # TODO: DELETE FROM application_services_regex WHERE id=this service + # TODO: INSERT INTO application_services_regex + # TODO: Update cache + pass + + def get_services_for_event(self, event): + return self.cache.get_services_for_event(event) @defer.inlineCallbacks - def get_app_service(self, as_token): + def get_app_service(self, as_token, from_cache=True): """Get the application service with the given token. Args: token (str): The application service token. + from_cache (bool): True to get this service from the cache, False to + check the database. Raises: - StoreError if there was a problem retrieving this. + StoreError if there was a problem retrieving this service. """ + + if from_cache: + for service in self.cache.services: + if service.token == as_token: + defer.returnValue(service) + return + defer.returnValue(None) + return + + + # TODO: This should be JOINed with the application_services_regex table. row = self._simple_select_one( "application_services", {"token": as_token}, ["url", "token"] @@ -101,3 +145,7 @@ class ApplicationServiceStore(SQLBaseStore): if not row: raise StoreError(400, "Bad application services token supplied.") defer.returnValue(row) + + def _populate_cache(self): + """Populates the ApplicationServiceCache from the database.""" + pass -- cgit 1.4.1 From ec3719b583c6fbbc56dbd313b858054e535ae733 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 17:15:06 +0000 Subject: Use ApplicationService when registering. --- synapse/handlers/appservice.py | 13 +++++-------- synapse/rest/appservice/v1/register.py | 8 ++++++-- synapse/storage/appservice.py | 16 ++++++---------- 3 files changed, 17 insertions(+), 20 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 25e1cece56..1890ca06aa 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -30,21 +30,18 @@ class ApplicationServicesHandler(BaseHandler): super(ApplicationServicesHandler, self).__init__(hs) @defer.inlineCallbacks - def register(self, base_url, token, namespaces): + def register(self, app_service): # check the token is recognised try: - app_service = yield self.store.get_app_service(token) - if not app_service: - raise StoreError + stored_service = yield self.store.get_app_service(app_service.token) + if not stored_service: + raise StoreError(404, "Not found") except StoreError: raise SynapseError( 403, "Unrecognised application services token. " "Consult the home server admin." ) - - # store this AS - - defer.returnValue("not_implemented_yet") + # TODO store this AS def unregister(self, token): yield self.store.unregister_app_service(token) diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 142f09a638..5786cf873e 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -18,6 +18,7 @@ from twisted.internet import defer from base import AppServiceRestServlet, as_path_pattern from synapse.api.errors import CodeMessageException, SynapseError +from synapse.storage.appservice import ApplicationService import json import logging @@ -58,7 +59,10 @@ class RegisterRestServlet(AppServiceRestServlet): self._parse_namespace(namespaces, params["namespaces"], "rooms") self._parse_namespace(namespaces, params["namespaces"], "aliases") - hs_token = yield self.handler.register(as_url, as_token, namespaces) + app_service = ApplicationService(as_token, as_url, namespaces) + + yield self.handler.register(app_service) + hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? defer.returnValue({ "hs_token": hs_token @@ -97,7 +101,7 @@ class UnregisterRestServlet(AppServiceRestServlet): except (KeyError, ValueError): raise SynapseError(400, "Missing required key: as_token(str)") - # TODO: pass to the appservice handler + yield self.handler.unregister(as_token) raise CodeMessageException(500, "Not implemented") diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index fbad17cb9e..f84f026b7b 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -116,8 +116,7 @@ class ApplicationServiceStore(SQLBaseStore): def get_services_for_event(self, event): return self.cache.get_services_for_event(event) - @defer.inlineCallbacks - def get_app_service(self, as_token, from_cache=True): + def get_app_service(self, token, from_cache=True): """Get the application service with the given token. Args: @@ -130,21 +129,18 @@ class ApplicationServiceStore(SQLBaseStore): if from_cache: for service in self.cache.services: - if service.token == as_token: - defer.returnValue(service) - return - defer.returnValue(None) - return - + if service.token == token: + return service + return None # TODO: This should be JOINed with the application_services_regex table. row = self._simple_select_one( - "application_services", {"token": as_token}, + "application_services", {"token": token}, ["url", "token"] ) if not row: raise StoreError(400, "Bad application services token supplied.") - defer.returnValue(row) + return row def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" -- cgit 1.4.1 From fbeaeb868960099c3682802275d5a222c0cc2d8b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 17:34:40 +0000 Subject: Log when ASes are registered/unregistered. --- synapse/handlers/appservice.py | 2 ++ synapse/storage/appservice.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 1890ca06aa..c9f56c41eb 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -31,6 +31,7 @@ class ApplicationServicesHandler(BaseHandler): @defer.inlineCallbacks def register(self, app_service): + logger.info("Register -> %s", app_service) # check the token is recognised try: stored_service = yield self.store.get_app_service(app_service.token) @@ -44,6 +45,7 @@ class ApplicationServicesHandler(BaseHandler): # TODO store this AS def unregister(self, token): + logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) def notify_interested_services(self, event): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index f84f026b7b..cd15843ba3 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -51,6 +51,9 @@ class ApplicationService(object): return True + def __str__(self): + return "ApplicationService: %s" % (self.__dict__,) + class ApplicationServiceCache(object): """Caches ApplicationServices and provides utility functions on top. @@ -83,7 +86,6 @@ class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.cache = ApplicationServiceCache() - self.clock = hs.get_clock() self._populate_cache() def unregister_app_service(self, token): -- cgit 1.4.1 From 54e513b4e6b5c644b9a2aeb02cef8258e87ae26a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 17:48:13 +0000 Subject: Move storage of user filters into real datastore layer; now have to mock it out in the REST-level tests --- synapse/api/filtering.py | 27 +++--------------- synapse/storage/__init__.py | 3 +- synapse/storage/filtering.py | 46 +++++++++++++++++++++++++++++++ tests/rest/client/v2_alpha/__init__.py | 9 ++++-- tests/rest/client/v2_alpha/test_filter.py | 21 ++++++++++++++ 5 files changed, 79 insertions(+), 27 deletions(-) create mode 100644 synapse/storage/filtering.py (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 014e2e1fc9..20b6951d47 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -16,37 +16,18 @@ from twisted.internet import defer -# TODO(paul) -_filters_for_user = {} - - class Filtering(object): def __init__(self, hs): super(Filtering, self).__init__() - self.hs = hs + self.store = hs.get_datastore() - @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): - filters = _filters_for_user.get(user_localpart, None) - - if not filters or filter_id >= len(filters): - raise KeyError() + return self.store.get_user_filter(user_localpart, filter_id) - # trivial yield to make it a generator so d.iC works - yield - defer.returnValue(filters[filter_id]) - - @defer.inlineCallbacks def add_user_filter(self, user_localpart, definition): - filters = _filters_for_user.setdefault(user_localpart, []) - - filter_id = len(filters) - filters.append(definition) - - # trivial yield, see above - yield - defer.returnValue(filter_id) + # TODO(paul): implement sanity checking of the definition + return self.store.add_user_filter(user_localpart, definition) # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4beb951b9f..efa63031bd 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,9 +30,9 @@ from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore from .media_repository import MediaRepositoryStore - from .state import StateStore from .signatures import SignatureStore +from .filtering import FilteringStore from syutil.base64util import decode_base64 from syutil.jsonutil import encode_canonical_json @@ -82,6 +82,7 @@ class DataStore(RoomMemberStore, RoomStore, DirectoryStore, KeyStore, StateStore, SignatureStore, EventFederationStore, MediaRepositoryStore, + FilteringStore, ): def __init__(self, hs): diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py new file mode 100644 index 0000000000..18e0e7c298 --- /dev/null +++ b/synapse/storage/filtering.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import SQLBaseStore + + +# TODO(paul) +_filters_for_user = {} + + +class FilteringStore(SQLBaseStore): + @defer.inlineCallbacks + def get_user_filter(self, user_localpart, filter_id): + filters = _filters_for_user.get(user_localpart, None) + + if not filters or filter_id >= len(filters): + raise KeyError() + + # trivial yield to make it a generator so d.iC works + yield + defer.returnValue(filters[filter_id]) + + @defer.inlineCallbacks + def add_user_filter(self, user_localpart, definition): + filters = _filters_for_user.setdefault(user_localpart, []) + + filter_id = len(filters) + filters.append(definition) + + # trivial yield, see above + yield + defer.returnValue(filter_id) diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index f59745e13c..3fe62d5ac6 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -39,9 +39,7 @@ class V2AlphaRestTestCase(unittest.TestCase): hs = HomeServer("test", db_pool=None, - datastore=Mock(spec=[ - "insert_client_ip", - ]), + datastore=self.make_datastore_mock(), http_client=None, resource_for_client=self.mock_resource, resource_for_federation=self.mock_resource, @@ -58,3 +56,8 @@ class V2AlphaRestTestCase(unittest.TestCase): for r in self.TO_REGISTER: r.register_servlets(hs, self.mock_resource) + + def make_datastore_mock(self): + return Mock(spec=[ + "insert_client_ip", + ]) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 8629a1aed6..1add727e6b 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -15,6 +15,8 @@ from twisted.internet import defer +from mock import Mock + from . import V2AlphaRestTestCase from synapse.rest.client.v2_alpha import filter @@ -24,6 +26,25 @@ class FilterTestCase(V2AlphaRestTestCase): USER_ID = "@apple:test" TO_REGISTER = [filter] + def make_datastore_mock(self): + datastore = super(FilterTestCase, self).make_datastore_mock() + + self._user_filters = {} + + def add_user_filter(user_localpart, definition): + filters = self._user_filters.setdefault(user_localpart, []) + filter_id = len(filters) + filters.append(definition) + return defer.succeed(filter_id) + datastore.add_user_filter = add_user_filter + + def get_user_filter(user_localpart, filter_id): + filters = self._user_filters[user_localpart] + return defer.succeed(filters[filter_id]) + datastore.get_user_filter = get_user_filter + + return datastore + @defer.inlineCallbacks def test_filter(self): (code, response) = yield self.mock_resource.trigger("POST", -- cgit 1.4.1 From 06cc1470129d443f71bfc81ba716f63b9505467d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 18:46:03 +0000 Subject: Initial stab at real SQL storage implementation of user filter definitions --- synapse/storage/__init__.py | 1 + synapse/storage/filtering.py | 49 +++++++++++++++++++++++++----------- synapse/storage/schema/filtering.sql | 24 ++++++++++++++++++ tests/api/test_filtering.py | 19 +++++++++++++- 4 files changed, 78 insertions(+), 15 deletions(-) create mode 100644 synapse/storage/schema/filtering.sql (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index efa63031bd..7c5631d014 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -61,6 +61,7 @@ SCHEMAS = [ "event_edges", "event_signatures", "media_repository", + "filtering", ] diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index 18e0e7c298..e98eaf8032 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -17,6 +17,8 @@ from twisted.internet import defer from ._base import SQLBaseStore +import json + # TODO(paul) _filters_for_user = {} @@ -25,22 +27,41 @@ _filters_for_user = {} class FilteringStore(SQLBaseStore): @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): - filters = _filters_for_user.get(user_localpart, None) - - if not filters or filter_id >= len(filters): - raise KeyError() + def_json = yield self._simple_select_one_onecol( + table="user_filters", + keyvalues={ + "user_id": user_localpart, + "filter_id": filter_id, + }, + retcol="definition", + allow_none=False, + ) - # trivial yield to make it a generator so d.iC works - yield - defer.returnValue(filters[filter_id]) + defer.returnValue(json.loads(def_json)) - @defer.inlineCallbacks def add_user_filter(self, user_localpart, definition): - filters = _filters_for_user.setdefault(user_localpart, []) + def_json = json.dumps(definition) + + # Need an atomic transaction to SELECT the maximal ID so far then + # INSERT a new one + def _do_txn(txn): + sql = ( + "SELECT MAX(filter_id) FROM user_filters " + "WHERE user_id = ?" + ) + txn.execute(sql, (user_localpart,)) + max_id = txn.fetchone()[0] + if max_id is None: + filter_id = 0 + else: + filter_id = max_id + 1 + + sql = ( + "INSERT INTO user_filters (user_id, filter_id, definition)" + "VALUES(?, ?, ?)" + ) + txn.execute(sql, (user_localpart, filter_id, def_json)) - filter_id = len(filters) - filters.append(definition) + return filter_id - # trivial yield, see above - yield - defer.returnValue(filter_id) + return self.runInteraction("add_user_filter", _do_txn) diff --git a/synapse/storage/schema/filtering.sql b/synapse/storage/schema/filtering.sql new file mode 100644 index 0000000000..795aca4afd --- /dev/null +++ b/synapse/storage/schema/filtering.sql @@ -0,0 +1,24 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS user_filters( + user_id TEXT, + filter_id INTEGER, + definition TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index fecadd1056..149948374d 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -53,16 +53,33 @@ class FilteringTestCase(unittest.TestCase): self.filtering = hs.get_filtering() + self.datastore = hs.get_datastore() + @defer.inlineCallbacks - def test_filter(self): + def test_add_filter(self): filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, definition={"type": ["m.*"]}, ) + self.assertEquals(filter_id, 0) + self.assertEquals({"type": ["m.*"]}, + (yield self.datastore.get_user_filter( + user_localpart=user_localpart, + filter_id=0, + )) + ) + + @defer.inlineCallbacks + def test_get_filter(self): + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + definition={"type": ["m.*"]}, + ) filter = yield self.filtering.get_user_filter( user_localpart=user_localpart, filter_id=filter_id, ) + self.assertEquals(filter, {"type": ["m.*"]}) -- cgit 1.4.1 From 8398f19bcea8fb0134b37efa303dc65b017d75ce Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 19:00:09 +0000 Subject: Created schema delta --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/v12.sql | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/v12.sql (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7c5631d014..00a04f565d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,7 +67,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 11 +SCHEMA_VERSION = 12 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql new file mode 100644 index 0000000000..795aca4afd --- /dev/null +++ b/synapse/storage/schema/delta/v12.sql @@ -0,0 +1,24 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS user_filters( + user_id TEXT, + filter_id INTEGER, + definition TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); -- cgit 1.4.1 From b19cf6a105ad87954e15cf01e60e14fec280db6d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 20:09:52 +0000 Subject: Wait for events if the incremental sync is empty and a timeout is given --- demo/start.sh | 2 +- synapse/handlers/sync.py | 19 ++++++++++++------- synapse/notifier.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/demo/start.sh b/demo/start.sh index bb2248770d..a7502e7a8d 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -16,7 +16,7 @@ if [ $# -eq 1 ]; then fi fi -for port in 8080 8081 8082; do +for port in 8080; do echo "Starting server on port $port... " https_port=$((port + 400)) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f8629a588f..9f5f73eab6 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -72,6 +72,7 @@ class SyncHandler(BaseHandler): self.event_sources = hs.get_event_sources() self.clock = hs.get_clock() + @defer.inlineCallbacks def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): """Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then @@ -80,15 +81,19 @@ class SyncHandler(BaseHandler): A Deferred SyncResult. """ if timeout == 0 or since_token is None: - return self.current_sync_for_user(sync_config, since_token) + result = yield self.current_sync_for_user(sync_config, since_token) + defer.returnValue(result) else: - def current_sync_callback(since_token): - return self.current_sync_for_user( - self, since_token, sync_config - ) - return self.notifier.wait_for_events( - sync_config.filter, since_token, current_sync_callback + def current_sync_callback(): + return self.current_sync_for_user(sync_config, since_token) + + rm_handler = self.hs.get_handlers().room_member_handler + room_ids = yield rm_handler.get_rooms_for_user(sync_config.user) + result = yield self.notifier.wait_for_events( + sync_config.user, room_ids, + sync_config.filter, timeout, current_sync_callback ) + defer.returnValue(result) def current_sync_for_user(self, sync_config, since_token=None): """Get the sync for client needed to match what the server has now. diff --git a/synapse/notifier.py b/synapse/notifier.py index 3aec1d4af2..922bf064d0 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext from synapse.util.async import run_on_reactor +from synapse.types import StreamToken import logging @@ -205,6 +206,53 @@ class Notifier(object): [notify(l).addErrback(eb) for l in listeners] ) + @defer.inlineCallbacks + def wait_for_events(self, user, rooms, filter, timeout, callback): + """Wait until the callback returns a non empty response or the + timeout fires. + """ + + deferred = defer.Deferred() + + from_token=StreamToken("s0","0","0") + + listener = [_NotificationListener( + user=user, + rooms=rooms, + from_token=from_token, + limit=1, + timeout=timeout, + deferred=deferred, + )] + + if timeout: + self._register_with_keys(listener[0]) + + result = yield callback() + if timeout: + timed_out = [False] + def _timeout_listener(): + timed_out[0] = True + listener[0].notify(self, [], from_token, from_token) + + self.clock.call_later(timeout/1000., _timeout_listener) + while not result and not timed_out[0]: + yield deferred + deferred = defer.Deferred() + listener[0] = _NotificationListener( + user=user, + rooms=rooms, + from_token=from_token, + limit=1, + timeout=timeout, + deferred=deferred, + ) + self._register_with_keys(listener[0]) + result = yield callback() + + defer.returnValue(result) + + def get_events_for(self, user, rooms, pagination_config, timeout): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any -- cgit 1.4.1 From e020574d65a994858ac53c45070ae5016090d2f3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 20:19:36 +0000 Subject: Fix Formatting --- synapse/handlers/sync.py | 13 +++++-------- synapse/notifier.py | 4 ++-- synapse/rest/client/v2_alpha/sync.py | 5 ++--- synapse/storage/stream.py | 1 - 4 files changed, 9 insertions(+), 14 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9f5f73eab6..82a2c6986a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -52,10 +52,10 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ class SyncResult(collections.namedtuple("SyncResult", [ - "next_batch", # Token for the next sync - "private_user_data", # List of private events for the user. - "public_user_data", # List of public events for all users. - "rooms", # RoomSyncResult for each room. + "next_batch", # Token for the next sync + "private_user_data", # List of private events for the user. + "public_user_data", # List of public events for all users. + "rooms", # RoomSyncResult for each room. ])): __slots__ = [] @@ -181,7 +181,6 @@ class SyncHandler(BaseHandler): limited=True, )) - @defer.inlineCallbacks def incremental_sync_with_gap(self, sync_config, since_token): """ Get the incremental delta needed to bring the client up to @@ -231,7 +230,6 @@ class SyncHandler(BaseHandler): next_batch=now_token, )) - @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, @@ -316,7 +314,6 @@ class SyncHandler(BaseHandler): state = () defer.returnValue(state) - def compute_state_delta(self, since_token, previous_state, current_state): """ Works out the differnce in state between the current state and the state the client got when it last performed a sync. @@ -327,7 +324,7 @@ class SyncHandler(BaseHandler): # after the previous sync, since we need to include those state # updates even if they occured logically before the previous event. # TODO(mjark) Check for new redactions in the state events. - previous_dict = {event.event_id:event for event in previous_state} + previous_dict = {event.event_id: event for event in previous_state} state_delta = [] for event in current_state: if event.event_id not in previous_dict: diff --git a/synapse/notifier.py b/synapse/notifier.py index 922bf064d0..e3b6ead620 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -214,7 +214,7 @@ class Notifier(object): deferred = defer.Deferred() - from_token=StreamToken("s0","0","0") + from_token = StreamToken("s0", "0", "0") listener = [_NotificationListener( user=user, @@ -231,6 +231,7 @@ class Notifier(object): result = yield callback() if timeout: timed_out = [False] + def _timeout_listener(): timed_out[0] = True listener[0].notify(self, [], from_token, from_token) @@ -252,7 +253,6 @@ class Notifier(object): defer.returnValue(result) - def get_events_for(self, user, rooms, pagination_config, timeout): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index cc667ebafc..0c17208cd3 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -68,7 +68,6 @@ class SyncRestServlet(RestServlet): } """ - PATTERN = client_v2_pattern("/sync$") ALLOWED_SORT = set(["timeline,asc", "timeline,desc"]) ALLOWED_PRESENCE = set(["online", "offline", "idle"]) @@ -114,12 +113,12 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, - device="TODO", # TODO(mjark) Get the device_id from access_token + device="TODO", # TODO(mjark) Get the device_id from access_token gap=gap, limit=limit, sort=sort, backfill=backfill, - filter="TODO", # TODO(mjark) Add the filter to the config. + filter="TODO", # TODO(mjark) Add the filter to the config. ) if since is not None: diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 06aca1a4e5..db1816ea84 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -289,7 +289,6 @@ class StreamStore(SQLBaseStore): " LIMIT ?" ) - def get_recent_events_for_room_txn(txn): if from_token is None: txn.execute(sql, (room_id, end_token.stream, limit,)) -- cgit 1.4.1 From b46fa8603e8e0726ea12310a77ade5cea59c3ae2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 09:17:48 +0000 Subject: Remove unused import --- synapse/storage/appservice.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index cd15843ba3..533fac4972 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from synapse.api.errors import StoreError from ._base import SQLBaseStore -- cgit 1.4.1 From e32ded7b3e290067f95497ceee0ac04842ba9cdf Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 10:09:54 +0000 Subject: Add matrix.org as a trusted ID server because it's now passed through on ports 80/443 and the web client defaults to that now. Fixes email validation (including signing up with an email address). --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 732652c228..66a89c10b2 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -163,7 +163,7 @@ class RegistrationHandler(BaseHandler): # each request httpCli = SimpleHttpClient(self.hs) # XXX: make this configurable! - trustedIdServers = ['matrix.org:8090'] + trustedIdServers = ['matrix.org:8090', 'matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) -- cgit 1.4.1 From 273b12729b99addf4474c9092f44ff300fd8153b Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 11:55:49 +0000 Subject: Reset badge count to zero when last active time is bumped --- synapse/handlers/presence.py | 5 +++++ synapse/push/__init__.py | 19 +++++++++++++++++++ synapse/push/httppusher.py | 38 +++++++++++++++++++++++++++++++++++--- synapse/push/pusherpool.py | 17 +++++++++++++++++ 4 files changed, 76 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 8aeed99274..24d901b51f 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -86,6 +86,10 @@ class PresenceHandler(BaseHandler): "changed_presencelike_data", self.changed_presencelike_data ) + # outbound signal from the presence module to advertise when a user's + # presence has changed + distributor.declare("user_presence_changed") + self.distributor = distributor self.federation = hs.get_replication_layer() @@ -603,6 +607,7 @@ class PresenceHandler(BaseHandler): room_ids=room_ids, statuscache=statuscache, ) + yield self.distributor.fire("user_presence_changed", user, statuscache) @defer.inlineCallbacks def _push_presence_remote(self, user, destination, state=None): diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index b6d01a82a0..4862d0de27 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -54,6 +54,9 @@ class Pusher(object): self.failing_since = failing_since self.alive = True + # The last value of last_active_time that we saw + self.last_last_active_time = 0 + @defer.inlineCallbacks def _actions_for_event(self, ev): """ @@ -273,6 +276,22 @@ class Pusher(object): """ pass + def reset_badge_count(self): + pass + + def presence_changed(self, state): + """ + We clear badge counts whenever a user's last_active time is bumped + This is by no means perfect but I think it's the best we can do + without read receipts. + """ + if 'last_active' in state.state: + last_active = state.state['last_active'] + if last_active > self.last_last_active_time: + logger.info("Resetting badge count for %s", self.user_name) + self.reset_badge_count() + self.last_last_active_time = last_active + def _value_for_dotted_key(dotted_key, event): parts = dotted_key.split(".") diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 22532fcc6a..d592bc2fd2 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -71,11 +71,12 @@ class HttpPusher(Pusher): # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? #'from_display_name': 'Steve Stevington' - #'counts': { -- we don't mark messages as read yet so + 'counts': { #-- we don't mark messages as read yet so # we have no way of knowing - # 'unread': 1, + # Just set the badge to 1 until we have read receipts + 'unread': 1, # 'missed_calls': 2 - # }, + }, 'devices': [ { 'app_id': self.app_id, @@ -111,3 +112,34 @@ class HttpPusher(Pusher): if 'rejected' in resp: rejected = resp['rejected'] defer.returnValue(rejected) + + @defer.inlineCallbacks + def reset_badge_count(self): + d = { + 'notification': { + 'id': '', + 'type': None, + 'from': '', + 'counts': { + 'unread': 0, + 'missed_calls': 0 + }, + 'devices': [ + { + 'app_id': self.app_id, + 'pushkey': self.pushkey, + 'pushkey_ts': long(self.pushkey_ts / 1000), + 'data': self.data_minus_url, + } + ] + } + } + try: + resp = yield self.httpCli.post_json_get_json(self.url, d) + except: + logger.exception("Failed to push %s ", self.url) + defer.returnValue(False) + rejected = [] + if 'rejected' in resp: + rejected = resp['rejected'] + defer.returnValue(rejected) \ No newline at end of file diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 2dfecf178b..65ab4f46e1 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -18,6 +18,7 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException +from synapse.api.constants import PresenceState import logging import json @@ -32,6 +33,22 @@ class PusherPool: self.pushers = {} self.last_pusher_started = -1 + distributor = self.hs.get_distributor() + distributor.observe( + "user_presence_changed", self.user_presence_changed + ) + + @defer.inlineCallbacks + def user_presence_changed(self, user, state): + user_name = user.to_string() + + # until we have read receipts, pushers use this to reset a user's + # badge counters to zero + for p in self.pushers.values(): + if p.user_name == user_name: + yield p.presence_changed(state) + + @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() -- cgit 1.4.1 From 42876969b99b6bad146b44a734e8d4a1a14d6835 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 11:59:38 +0000 Subject: Add basic application_services SQL, and hook up parts of the appservice store to read from it. --- synapse/handlers/appservice.py | 7 +- synapse/rest/appservice/v1/register.py | 4 +- synapse/storage/__init__.py | 1 + synapse/storage/appservice.py | 89 ++++++++++++++++++++++--- synapse/storage/schema/application_services.sql | 32 +++++++++ 5 files changed, 117 insertions(+), 16 deletions(-) create mode 100644 synapse/storage/schema/application_services.sql (limited to 'synapse') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index c9f56c41eb..8bd475cbfd 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -16,7 +16,7 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.errors import StoreError, SynapseError +from synapse.api.errors import Codes, StoreError, SynapseError import logging @@ -36,11 +36,12 @@ class ApplicationServicesHandler(BaseHandler): try: stored_service = yield self.store.get_app_service(app_service.token) if not stored_service: - raise StoreError(404, "Not found") + raise StoreError(404, "Application Service Not found") except StoreError: raise SynapseError( 403, "Unrecognised application services token. " - "Consult the home server admin." + "Consult the home server admin.", + errcode=Codes.FORBIDDEN ) # TODO store this AS diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 5786cf873e..e374d538e7 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -64,9 +64,9 @@ class RegisterRestServlet(AppServiceRestServlet): yield self.handler.register(app_service) hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? - defer.returnValue({ + defer.returnValue((200, { "hs_token": hs_token - }) + })) def _parse_namespace(self, target_ns, origin_ns, ns): if ns not in target_ns or ns not in origin_ns: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9431c1a32d..e86b981b47 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -62,6 +62,7 @@ SCHEMAS = [ "event_edges", "event_signatures", "media_repository", + "application_services" ] diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 533fac4972..5a0e47e0d4 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -12,12 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from synapse.api.errors import StoreError +import logging +from twisted.internet import defer from ._base import SQLBaseStore +logger = logging.getLogger(__name__) + + # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): """Defines an application service. @@ -30,7 +33,22 @@ class ApplicationService(object): if url: self.url = url if namespaces: - self.namespaces = namespaces + self._set_namespaces(namespaces) + + def _set_namespaces(self, namespaces): + # Sanity check that it is of the form: + # { + # users: ["regex",...], + # aliases: ["regex",...], + # rooms: ["regex",...], + # } + for ns in ["users", "rooms", "aliases"]: + if type(namespaces[ns]) != list: + raise ValueError("Bad namespace value for '%s'", ns) + for regex in namespaces[ns]: + if not isinstance(regex, basestring): + raise ValueError("Expected string regex for ns '%s'", ns) + self.namespaces = namespaces def is_interested(self, event): """Check if this service is interested in this event. @@ -133,15 +151,64 @@ class ApplicationServiceStore(SQLBaseStore): return service return None + # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. - row = self._simple_select_one( - "application_services", {"token": token}, - ["url", "token"] - ) - if not row: - raise StoreError(400, "Bad application services token supplied.") - return row + + @defer.inlineCallbacks def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" - pass + sql = ("SELECT * FROM application_services LEFT JOIN " + "application_services_regex ON application_services.id = " + "application_services_regex.as_id") + + namespace_enum = [ + "users", # 0 + "aliases", # 1 + "rooms" # 2 + ] + # SQL results in the form: + # [ + # { + # 'regex': "something", + # 'url': "something", + # 'namespace': enum, + # 'as_id': 0, + # 'token': "something", + # 'id': 0 + # } + # ] + services = {} + results = yield self._execute_and_decode(sql) + for res in results: + as_token = res["token"] + if as_token not in services: + # add the service + services[as_token] = { + "url": res["url"], + "token": as_token, + "namespaces": { + "users": [], + "aliases": [], + "rooms": [] + } + } + # add the namespace regex if one exists + ns_int = res["namespace"] + if ns_int is None: + continue + try: + services[as_token]["namespaces"][namespace_enum[ns_int]].append( + res["regex"] + ) + except IndexError: + logger.error("Bad namespace enum '%s'. %s", ns_int, res) + + for service in services.values(): + logger.info("Found application service: %s", service) + self.cache.services.append(ApplicationService( + service["token"], + service["url"], + service["namespaces"] + )) + diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql new file mode 100644 index 0000000000..6d245fc807 --- /dev/null +++ b/synapse/storage/schema/application_services.sql @@ -0,0 +1,32 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT, + token TEXT, + UNIQUE(token) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id INTEGER PRIMARY KEY AUTOINCREMENT, + as_id INTEGER NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex TEXT, + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + + + -- cgit 1.4.1 From fb532d84250e7d112461c62b3ec2443cc61d1348 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:06:09 +0000 Subject: Unused import --- synapse/push/pusherpool.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 65ab4f46e1..e69ee2cf17 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -18,7 +18,6 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException -from synapse.api.constants import PresenceState import logging import json -- cgit 1.4.1 From ca7240a2f06b6f6797c857b5cc4d8998a1a46557 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:17:55 +0000 Subject: Update copyright --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 4862d0de27..fd8d6d4bbd 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -- cgit 1.4.1 From 6df6f5e0840e6a2b878f62c1656834bfbd5bcd62 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:56:35 +0000 Subject: Redundant bracketing & missed space --- synapse/push/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index fd8d6d4bbd..8ddc833577 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -187,8 +187,8 @@ class Pusher(object): # for sanity, we only remove the pushkey if it # was the one we actually sent... logger.warn( - ("Ignoring rejected pushkey %s because we" + - "didn't send it"), (pk,) + ("Ignoring rejected pushkey %s because we " + + "didn't send it"), pk ) else: logger.info( -- cgit 1.4.1 From e1ca0f1396425db5c80a975bb83596c7384484ef Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:58:32 +0000 Subject: Brackets rather than slashes at end --- synapse/push/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8ddc833577..8a3e8fd2a3 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -226,9 +226,9 @@ class Pusher(object): self.failing_since ) - if self.failing_since and \ - self.failing_since < \ - self.clock.time_msec() - Pusher.GIVE_UP_AFTER: + if (self.failing_since and + self.failing_since < + self.clock.time_msec() - Pusher.GIVE_UP_AFTER): # we really only give up so that if the URL gets # fixed, we don't suddenly deliver a load # of old notifications. -- cgit 1.4.1 From 03149ad23ace1f200993a74b998a6986d70c9420 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:01:24 +0000 Subject: More code style things --- synapse/push/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8a3e8fd2a3..b79f2d4b27 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -187,7 +187,7 @@ class Pusher(object): # for sanity, we only remove the pushkey if it # was the one we actually sent... logger.warn( - ("Ignoring rejected pushkey %s because we " + + ("Ignoring rejected pushkey %s because we " "didn't send it"), pk ) else: @@ -234,7 +234,8 @@ class Pusher(object): # of old notifications. logger.warn("Giving up on a notification to user %s, " "pushkey %s", - self.user_name, self.pushkey) + self.user_name, self.pushkey + ) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token( @@ -256,7 +257,7 @@ class Pusher(object): self.user_name, self.clock.time_msec() - self.failing_since, self.backoff_delay - ) + ) yield synapse.util.async.sleep(self.backoff_delay / 1000.0) self.backoff_delay *= 2 if self.backoff_delay > Pusher.MAX_BACKOFF: -- cgit 1.4.1 From 20c47383dca7d68608c903fe1f856756fd3da057 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:10:46 +0000 Subject: Oops, bad merge: needed to change the base class of the rest servlets too. --- synapse/push/__init__.py | 5 +++-- synapse/rest/client/v1/push_rule.py | 4 ++-- synapse/rest/client/v1/pusher.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index b79f2d4b27..1cac5fff4e 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -269,8 +269,9 @@ class Pusher(object): def dispatch_push(self, p, tweaks): """ Overridden by implementing classes to actually deliver the notification - :param p: The event to notify for as a single event from the event stream - :return: If the notification was delivered, an array containing any + Args: + p The event to notify for as a single event from the event stream + Returns: If the notification was delivered, an array containing any pushkeys that were rejected by the push gateway. False if the notification could not be delivered (ie. should be retried). diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 50bf5b9008..00fed42f44 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -17,13 +17,13 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError, \ StoreError -from base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException import json -class PushRuleRestServlet(RestServlet): +class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") PRIORITY_CLASS_MAP = { 'underride': 0, diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 4659c9b1d9..80a11890a3 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -17,12 +17,12 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes from synapse.push import PusherConfigException -from base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern import json -class PusherRestServlet(RestServlet): +class PusherRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushers/set$") @defer.inlineCallbacks -- cgit 1.4.1 From 5f2665320fbe4392c9f487820c697691463cf709 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:11:45 +0000 Subject: It is 2015 --- synapse/push/httppusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index d592bc2fd2..e12b946727 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -- cgit 1.4.1 From 6fde707addc29b6717fda9617a127093fb5b08ae Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:14:49 +0000 Subject: doc style fix --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 1cac5fff4e..10ac890482 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -270,7 +270,7 @@ class Pusher(object): """ Overridden by implementing classes to actually deliver the notification Args: - p The event to notify for as a single event from the event stream + p: The event to notify for as a single event from the event stream Returns: If the notification was delivered, an array containing any pushkeys that were rejected by the push gateway. False if the notification could not be delivered (ie. -- cgit 1.4.1 From dd3abbd61fba7d3ea9fe300a309202f0eb4c5fbd Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:22:39 +0000 Subject: 2015 --- synapse/push/pusherpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index e69ee2cf17..716d9514cd 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -- cgit 1.4.1 From 30fbba168b0f9bf8b38594b3badb9af463167485 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:23:16 +0000 Subject: Easy on the newlines --- synapse/push/pusherpool.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 716d9514cd..856defedac 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -47,7 +47,6 @@ class PusherPool: if p.user_name == user_name: yield p.presence_changed(state) - @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() -- cgit 1.4.1 From 4fbf2328c29ca8f8a7918f6763b91787e55f37f8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:24:28 +0000 Subject: Unnecessary new line --- synapse/rest/client/v1/__init__.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index 96a9a474f1..d8d01cdd16 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from . import ( room, events, register, login, profile, presence, initial_sync, directory, voip, admin, pusher, push_rule -- cgit 1.4.1 From 6741c3dbd90b6406d7eba8445868cdd8fd2ec6a5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:26:03 +0000 Subject: Brackets are nicer --- synapse/rest/client/v1/push_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 00fed42f44..b03d804d82 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -33,8 +33,8 @@ class PushRuleRestServlet(ClientV1RestServlet): 'override': 4 } PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} - SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR =\ - "Unrecognised request: You probably wanted a trailing slash" + SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( + "Unrecognised request: You probably wanted a trailing slash") def rule_spec_from_path(self, path): if len(path) < 2: -- cgit 1.4.1 From d93ce29a86b79478bfb2011c9b8d3c8ec7d0bdd0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:27:01 +0000 Subject: Ah, the comma of doom. --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index b03d804d82..c6085370a4 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -101,7 +101,7 @@ class PushRuleRestServlet(ClientV1RestServlet): if pat.strip("*?[]") == pat: # no special glob characters so we assume the user means # 'contains this string' rather than 'is this string' - pat = "*%s*" % (pat) + pat = "*%s*" % (pat,) conditions = [{ 'kind': 'event_match', 'key': 'content.body', -- cgit 1.4.1 From 032f8d4ed3a5b4015087b7a97295da9b580a95b3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:33:15 +0000 Subject: Another superfluous newline --- synapse/rest/client/v1/push_rule.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index c6085370a4..3a08bdd9af 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -213,7 +213,6 @@ class PushRuleRestServlet(ClientV1RestServlet): else: raise - @defer.inlineCallbacks def on_GET(self, request): user = yield self.auth.get_user_by_req(request) -- cgit 1.4.1 From 8807f4170eaf2515407b656b7eb6fe7e8fc93796 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:35:00 +0000 Subject: Better style --- synapse/rest/client/v1/push_rule.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 3a08bdd9af..52f2b19bbe 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -242,8 +242,11 @@ class PushRuleRestServlet(ClientV1RestServlet): continue if instance_handle not in rules['device']: rules['device'][instance_handle] = {} - rules['device'][instance_handle] = \ - _add_empty_priority_class_arrays(rules['device'][instance_handle]) + rules['device'][instance_handle] = ( + _add_empty_priority_class_arrays( + rules['device'][instance_handle] + ) + ) rulearray = rules['device'][instance_handle][template_name] else: -- cgit 1.4.1 From 3cb5b73c0dd10ba9020547a81864846c70d4e709 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:37:55 +0000 Subject: Unnecessary newline. --- synapse/rest/client/v1/push_rule.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 52f2b19bbe..550554c18c 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -290,7 +290,6 @@ class PushRuleRestServlet(ClientV1RestServlet): def on_OPTIONS(self, _): return 200, {} - def _add_empty_priority_class_arrays(d): for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): d[pc] = [] -- cgit 1.4.1 From 289a2498743caae7e0c3366a1c1f7855d48d9a8e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:39:03 +0000 Subject: Unnecessary newlines. --- synapse/rest/client/v1/push_rule.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 550554c18c..dbcac3d4e1 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -295,7 +295,6 @@ def _add_empty_priority_class_arrays(d): d[pc] = [] return d - def _instance_handle_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the @@ -326,7 +325,6 @@ def _filter_ruleset_with_path(ruleset, path): return r raise NotFoundError - def _priority_class_from_spec(spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) @@ -337,7 +335,6 @@ def _priority_class_from_spec(spec): return pc - def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device @@ -346,7 +343,6 @@ def _priority_class_to_template_name(pc): else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] - def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: @@ -363,7 +359,6 @@ def _rule_to_template(rule): ret["pattern"] = thecond["pattern"] return ret - def _strip_device_condition(rule): for i,c in enumerate(rule['conditions']): if c['kind'] == 'device': -- cgit 1.4.1 From 2cfdfee572dc037b65571ab72efcb3223c7d8d11 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:41:51 +0000 Subject: spaces --- synapse/rest/client/v1/push_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index dbcac3d4e1..2b33bdac08 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -346,7 +346,7 @@ def _priority_class_to_template_name(pc): def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: - return {k:rule[k] for k in ["rule_id", "conditions", "actions"]} + return {k: rule[k] for k in ["rule_id", "conditions", "actions"]} elif template_name in ["sender", "room"]: return {k: rule[k] for k in ["rule_id", "actions"]} elif template_name == 'content': @@ -355,7 +355,7 @@ def _rule_to_template(rule): thecond = rule["conditions"][0] if "pattern" not in thecond: return None - ret = {k:rule[k] for k in ["rule_id", "actions"]} + ret = {k: rule[k] for k in ["rule_id", "actions"]} ret["pattern"] = thecond["pattern"] return ret -- cgit 1.4.1 From 0cbb6b0f5235e4501a0fb360e881d152644a17cd Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:44:41 +0000 Subject: Google doc style --- synapse/storage/_base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4f172d3967..809c81f47f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -195,10 +195,11 @@ class SQLBaseStore(object): def _simple_upsert(self, table, keyvalues, values): """ - :param table: The table to upsert into - :param keyvalues: Dict of the unique key tables and their new values - :param values: Dict of all the nonunique columns and their new values - :return: A deferred + Args: + table (str): The table to upsert into + keyvalues (dict): The unique key tables and their new values + values (dict): The nonunique columns and their new values + Returns: A deferred """ return self.runInteraction( "_simple_upsert", -- cgit 1.4.1 From fb0928097a0dc1606aebb9aed8f070bcea304178 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:48:07 +0000 Subject: More magic commas (including the place I copied it from...) --- synapse/storage/_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 809c81f47f..9261c999cb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -210,8 +210,8 @@ class SQLBaseStore(object): # Try to update sql = "UPDATE %s SET %s WHERE %s" % ( table, - ", ".join("%s = ?" % (k) for k in values), - " AND ".join("%s = ?" % (k) for k in keyvalues) + ", ".join("%s = ?" % (k,) for k in values), + " AND ".join("%s = ?" % (k,) for k in keyvalues) ) sqlargs = values.values() + keyvalues.values() logger.debug( @@ -390,8 +390,8 @@ class SQLBaseStore(object): if updatevalues: update_sql = "UPDATE %s SET %s WHERE %s" % ( table, - ", ".join("%s = ?" % (k) for k in updatevalues), - " AND ".join("%s = ?" % (k) for k in keyvalues) + ", ".join("%s = ?" % (k,) for k in updatevalues), + " AND ".join("%s = ?" % (k,) for k in keyvalues) ) def func(txn): -- cgit 1.4.1 From 6d485dd1c727e7ecfe3991066bd058794ae05051 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:48:42 +0000 Subject: unnecessary newlines --- synapse/storage/_base.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9261c999cb..4e8bd3faa9 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -237,8 +237,6 @@ class SQLBaseStore(object): ) txn.execute(sql, allvalues.values()) - - def _simple_select_one(self, table, keyvalues, retcols, allow_none=False): """Executes a SELECT query on the named table, which is expected to -- cgit 1.4.1 From 445ad9941ea2e4038846aa6fed456e3250ae49b1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:49:59 +0000 Subject: Redundant parens --- synapse/storage/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index ca04f2ccee..f5a736be44 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -92,7 +92,7 @@ class PushRuleStore(SQLBaseStore): res = txn.fetchall() if not res: raise RuleNotFoundException("before/after rule not found: %s" % (relative_to_rule)) - (priority_class, base_rule_priority) = res[0] + priority_class, base_rule_priority = res[0] if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: raise InconsistentRuleException( -- cgit 1.4.1 From 93aac9bb7b3023e6c82961b1cdd655a48ec567fb Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:51:01 +0000 Subject: Newline --- synapse/storage/push_rule.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index f5a736be44..48105234f6 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -184,6 +184,7 @@ class PushRuleStore(SQLBaseStore): } ) + class RuleNotFoundException(Exception): pass -- cgit 1.4.1 From e78dd332928c111c8a62985bce0a3c1c5631244e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:52:58 +0000 Subject: Use %s instead of + --- synapse/storage/push_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 48105234f6..0342996ed1 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -85,8 +85,8 @@ class PushRuleStore(SQLBaseStore): # get the priority of the rule we're inserting after/before sql = ( - "SELECT priority_class, priority FROM "+PushRuleTable.table_name+ - " WHERE user_name = ? and rule_id = ?" + "SELECT priority_class, priority FROM ? " + "WHERE user_name = ? and rule_id = ?" % (PushRuleTable.table_name,) ) txn.execute(sql, (user_name, relative_to_rule)) res = txn.fetchall() -- cgit 1.4.1 From c59bcabf0b5c0ab78c0f89da75b031993c4660d9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 15:36:21 +0000 Subject: Return the device_id from get_auth_by_req --- synapse/api/auth.py | 7 +++++-- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/directory.py | 4 ++-- synapse/rest/client/v1/events.py | 4 ++-- synapse/rest/client/v1/initial_sync.py | 2 +- synapse/rest/client/v1/presence.py | 8 ++++---- synapse/rest/client/v1/profile.py | 4 ++-- synapse/rest/client/v1/room.py | 24 ++++++++++++------------ synapse/rest/client/v1/voip.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/rest/media/v1/upload_resource.py | 2 +- tests/rest/client/v1/test_presence.py | 2 +- tests/rest/client/v1/test_profile.py | 2 +- 13 files changed, 34 insertions(+), 31 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a342a0e0da..292e9e2a80 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -290,7 +290,9 @@ class Auth(object): Args: request - An HTTP request with an access_token query parameter. Returns: - UserID : User ID object of the user making the request + Tuple of UserID and device string: + User ID object of the user making the request + Device ID string of the device the user is using Raises: AuthError if no user by that token exists or the token is invalid. """ @@ -299,6 +301,7 @@ class Auth(object): access_token = request.args["access_token"][0] user_info = yield self.get_user_by_token(access_token) user = user_info["user"] + device_id = user_info["device_id"] ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders( @@ -314,7 +317,7 @@ class Auth(object): user_agent=user_agent ) - defer.returnValue(user) + defer.returnValue((user, device_id)) except KeyError: raise AuthError(403, "Missing access token.") diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 1051d96f96..6cfce1a479 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -31,7 +31,7 @@ class WhoisRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) if not is_admin and target_user != auth_user: diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 15ae8749b8..ef853af411 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -45,7 +45,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) if not "room_id" in content: @@ -85,7 +85,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request, room_alias): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index a0d051227b..e58ee46fcd 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -34,7 +34,7 @@ class EventStreamRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) try: handler = self.handlers.event_stream_handler pagin_config = PaginationConfig.from_request(request) @@ -71,7 +71,7 @@ class EventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, event_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 357fa845b4..78d30abbf8 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -25,7 +25,7 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index b6c207e662..74669274a7 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -32,7 +32,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( @@ -42,7 +42,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = {} @@ -77,7 +77,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): @@ -97,7 +97,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 24f8d56952..f04abb2c26 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -37,7 +37,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: @@ -70,7 +70,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 58b09b6fc1..c8c34b4801 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -62,7 +62,7 @@ class RoomCreateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) room_config = self.get_room_config(request) info = yield self.make_room(room_config, auth_user, None) @@ -125,7 +125,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) msg_handler = self.handlers.message_handler data = yield msg_handler.get_room_data( @@ -143,7 +143,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, event_type, state_key): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -173,7 +173,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, event_type): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -216,7 +216,7 @@ class JoinRoomAliasServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_identifier): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) # the identifier could be a room alias or a room id. Try one then the # other if it fails to parse, without swallowing other valid @@ -283,7 +283,7 @@ class RoomMemberListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) handler = self.handlers.room_member_handler members = yield handler.get_room_members_as_pagination_chunk( room_id=room_id, @@ -311,7 +311,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request( request, default_limit=10, ) @@ -335,7 +335,7 @@ class RoomStateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) handler = self.handlers.message_handler # Get all the current state for this room events = yield handler.get_state_events( @@ -351,7 +351,7 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, @@ -396,7 +396,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, membership_action): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -445,7 +445,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, event_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -483,7 +483,7 @@ class RoomTypingRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) target_user = UserID.from_string(urllib.unquote(user_id)) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 822d863ce6..42d8e30bab 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -28,7 +28,7 @@ class VoipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) turnUris = self.hs.config.turn_uris turnSecret = self.hs.config.turn_shared_secret diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index 79ae0e3d74..311ab89edb 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -66,7 +66,7 @@ class ContentRepoResource(resource.Resource): @defer.inlineCallbacks def map_request_to_name(self, request): # auth the user - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) # namespace all file uploads on the user prefix = base64.urlsafe_b64encode( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index b1718a630b..6bed8a8efa 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -42,7 +42,7 @@ class UploadResource(BaseMediaResource): @defer.inlineCallbacks def _async_render_POST(self, request): try: - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 65d5cc4916..a4f2abf213 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -282,7 +282,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.get_clock().time_msec.return_value = 1000000 def _get_user_by_req(req=None): - return UserID.from_string(myid) + return (UserID.from_string(myid), "") hs.get_auth().get_user_by_req = _get_user_by_req diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 39cd68d829..6a2085276a 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -58,7 +58,7 @@ class ProfileTestCase(unittest.TestCase): ) def _get_user_by_req(request=None): - return UserID.from_string(myid) + return (UserID.from_string(myid), "") hs.get_auth().get_user_by_req = _get_user_by_req -- cgit 1.4.1 From 60b143a52e69751a406ea83cdab58f4045cdd9d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 15:48:28 +0000 Subject: Move pushers delta to v12 and bump schema version --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/v10.sql | 46 ------------------------------------ synapse/storage/schema/delta/v12.sql | 46 ++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 47 deletions(-) delete mode 100644 synapse/storage/schema/delta/v10.sql create mode 100644 synapse/storage/schema/delta/v12.sql (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2534d109fd..277581b4e2 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -69,7 +69,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 11 +SCHEMA_VERSION = 12 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql deleted file mode 100644 index 8c4dfd5c1b..0000000000 --- a/synapse/storage/schema/delta/v10.sql +++ /dev/null @@ -1,46 +0,0 @@ -/* Copyright 2014 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- Push notification endpoints that users have configured -CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - instance_handle varchar(32) NOT NULL, - kind varchar(8) NOT NULL, - app_id varchar(64) NOT NULL, - app_display_name varchar(64) NOT NULL, - device_display_name varchar(128) NOT NULL, - pushkey blob NOT NULL, - ts BIGINT NOT NULL, - lang varchar(8), - data blob, - last_token TEXT, - last_success BIGINT, - failing_since BIGINT, - FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (app_id, pushkey) -); - -CREATE TABLE IF NOT EXISTS push_rules ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, - priority_class TINYINT NOT NULL, - priority INTEGER NOT NULL DEFAULT 0, - conditions TEXT NOT NULL, - actions TEXT NOT NULL, - UNIQUE(user_name, rule_id) -); - -CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql new file mode 100644 index 0000000000..8c4dfd5c1b --- /dev/null +++ b/synapse/storage/schema/delta/v12.sql @@ -0,0 +1,46 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + instance_handle varchar(32) NOT NULL, + kind varchar(8) NOT NULL, + app_id varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + ts BIGINT NOT NULL, + lang varchar(8), + data blob, + last_token TEXT, + last_success BIGINT, + failing_since BIGINT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (app_id, pushkey) +); + +CREATE TABLE IF NOT EXISTS push_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); -- cgit 1.4.1 From 0ef5bfd6a9eaaae14e199997658b3d0006abd854 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 28 Jan 2015 16:16:53 +0000 Subject: Start implementing auth conflict res --- synapse/api/auth.py | 38 +++--- synapse/api/constants.py | 6 + synapse/federation/federation_client.py | 39 ++++++ synapse/handlers/federation.py | 211 ++++++++++++++++++++++++++------ synapse/storage/rejections.py | 10 ++ synapse/storage/schema/im.sql | 1 + 6 files changed, 253 insertions(+), 52 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a342a0e0da..461faa8c78 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -353,9 +353,23 @@ class Auth(object): def add_auth_events(self, builder, context): yield run_on_reactor() - if builder.type == EventTypes.Create: - builder.auth_events = [] - return + auth_ids = self.compute_auth_events(builder, context) + + auth_events_entries = yield self.store.add_event_hashes( + auth_ids + ) + + builder.auth_events = auth_events_entries + + context.auth_events = { + k: v + for k, v in context.current_state.items() + if v.event_id in auth_ids + } + + def compute_auth_events(self, event, context): + if event.type == EventTypes.Create: + return [] auth_ids = [] @@ -368,7 +382,7 @@ class Auth(object): key = (EventTypes.JoinRules, "", ) join_rule_event = context.current_state.get(key) - key = (EventTypes.Member, builder.user_id, ) + key = (EventTypes.Member, event.user_id, ) member_event = context.current_state.get(key) key = (EventTypes.Create, "", ) @@ -382,8 +396,8 @@ class Auth(object): else: is_public = False - if builder.type == EventTypes.Member: - e_type = builder.content["membership"] + if event.type == EventTypes.Member: + e_type = event.content["membership"] if e_type in [Membership.JOIN, Membership.INVITE]: if join_rule_event: auth_ids.append(join_rule_event.event_id) @@ -398,17 +412,7 @@ class Auth(object): if member_event.content["membership"] == Membership.JOIN: auth_ids.append(member_event.event_id) - auth_events_entries = yield self.store.add_event_hashes( - auth_ids - ) - - builder.auth_events = auth_events_entries - - context.auth_events = { - k: v - for k, v in context.current_state.items() - if v.event_id in auth_ids - } + return auth_ids @log_function def _can_send_event(self, event, auth_events): diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 7ee6dcc46e..0d3fc629af 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -74,3 +74,9 @@ class EventTypes(object): Message = "m.room.message" Topic = "m.room.topic" Name = "m.room.name" + + +class RejectedReason(object): + AUTH_ERROR = "auth_error" + REPLACED = "replaced" + NOT_ANCESTOR = "not_ancestor" diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 91b44cd8b3..ebcd593506 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -331,6 +331,45 @@ class FederationClient(object): defer.returnValue(pdu) + @defer.inlineCallbacks + def query_auth(self, destination, room_id, event_id, local_auth): + """ + Params: + destination (str) + event_it (str) + local_auth (list) + """ + time_now = self._clock.time_msec() + + send_content = { + "auth_chain": [e.get_pdu_json(time_now) for e in local_auth], + } + + code, content = yield self.transport_layer.send_invite( + destination=destination, + room_id=room_id, + event_id=event_id, + content=send_content, + ) + + auth_chain = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content["auth_chain"] + ] + + missing = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content.get("missing", []) + ] + + ret = { + "auth_chain": auth_chain, + "rejects": content.get("rejects", []), + "missing": missing, + } + + defer.returnValue(ret) + def event_from_pdu_json(self, pdu_json, outlier=False): event = FrozenEvent( pdu_json diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index bcdcc90a18..97e3c503b9 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,19 +17,16 @@ from ._base import BaseHandler -from synapse.events.utils import prune_event from synapse.api.errors import ( - AuthError, FederationError, SynapseError, StoreError, + AuthError, FederationError, StoreError, ) -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, RejectedReason from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import ( - compute_event_signature, check_event_content_hash, - add_hashes_and_signatures, + compute_event_signature, add_hashes_and_signatures, ) from synapse.types import UserID -from syutil.jsonutil import encode_canonical_json from twisted.internet import defer @@ -113,33 +110,6 @@ class FederationHandler(BaseHandler): logger.debug("Processing event: %s", event.event_id) - redacted_event = prune_event(event) - - redacted_pdu_json = redacted_event.get_pdu_json() - try: - yield self.keyring.verify_json_for_server( - event.origin, redacted_pdu_json - ) - except SynapseError as e: - logger.warn( - "Signature check failed for %s redacted to %s", - encode_canonical_json(pdu.get_pdu_json()), - encode_canonical_json(redacted_pdu_json), - ) - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event.event_id, - ) - - if not check_event_content_hash(event): - logger.warn( - "Event content has been tampered, redacting %s, %s", - event.event_id, encode_canonical_json(event.get_dict()) - ) - event = redacted_event - logger.debug("Event: %s", event) # FIXME (erikj): Awful hack to make the case where we are not currently @@ -180,7 +150,6 @@ class FederationHandler(BaseHandler): if state: for e in state: - logging.info("A :) %r", e) e.internal_metadata.outlier = True try: yield self._handle_new_event(e) @@ -747,7 +716,20 @@ class FederationHandler(BaseHandler): event.event_id, event.signatures, ) - self.auth.check(event, auth_events=context.auth_events) + try: + self.auth.check(event, auth_events=context.auth_events) + except AuthError: + # TODO: Store rejection. + context.rejected = RejectedReason.AUTH_ERROR + + yield self.store.persist_event( + event, + context=context, + backfilled=backfilled, + is_new_state=False, + current_state=current_state, + ) + raise logger.debug( "_handle_new_event: Before persist_event: %s, sigs: %s", @@ -768,3 +750,162 @@ class FederationHandler(BaseHandler): ) defer.returnValue(context) + + @defer.inlineCallbacks + def do_auth(self, origin, event, context): + for e_id, _ in event.auth_events: + pass + + auth_events = set(e_id for e_id, _ in event.auth_events) + current_state = set(e.event_id for e in context.auth_events.values()) + + missing_auth = auth_events - current_state + + if missing_auth: + # Do auth conflict res. + + # 1. Get what we think is the auth chain. + auth_ids = self.auth.compute_auth_events(event, context) + local_auth_chain = yield self.store.get_auth_chain(auth_ids) + + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) + + # 3. Process any remote auth chain events we haven't seen. + for e in result.get("missing", []): + # TODO. + pass + + # 4. Look at rejects and their proofs. + # TODO. + + try: + self.auth.check(event, auth_events=context.auth_events) + except AuthError: + raise + + @defer.inlineCallbacks + def construct_auth_difference(self, local_auth, remote_auth): + """ Given a local and remote auth chain, find the differences. This + assumes that we have already processed all events in remote_auth + + Params: + local_auth (list) + remote_auth (list) + + Returns: + dict + """ + + # TODO: Make sure we are OK with local_auth or remote_auth having more + # auth events in them than strictly necessary. + + def sort_fun(ev): + return ev.depth, ev.event_id + + # We find the differences by starting at the "bottom" of each list + # and iterating up on both lists. The lists are ordered by depth and + # then event_id, we iterate up both lists until we find the event ids + # don't match. Then we look at depth/event_id to see which side is + # missing that event, and iterate only up that list. Repeat. + + remote_list = list(remote_auth) + remote_list.sort(key=sort_fun) + + local_list = list(local_auth) + local_list.sort(key=sort_fun) + + local_iter = iter(local_list) + remote_iter = iter(remote_list) + + current_local = local_iter.next() + current_remote = remote_iter.next() + + def get_next(it, opt=None): + return it.next() if it.has_next() else opt + + missing_remotes = [] + missing_locals = [] + while current_local and current_remote: + if current_remote is None: + missing_locals.append(current_local) + current_local = get_next(local_iter) + continue + + if current_local is None: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + if current_local.event_id == current_remote.event_id: + current_local = get_next(local_iter) + current_remote = get_next(remote_iter) + continue + + if current_local.depth < current_remote.depth: + missing_locals.append(current_local) + current_local = get_next(local_iter) + continue + + if current_local.depth > current_remote.depth: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + # They have the same depth, so we fall back to the event_id order + if current_local.event_id < current_remote.event_id: + missing_locals.append(current_local) + current_local = get_next(local_iter) + + if current_local.event_id > current_remote.event_id: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + # missing locals should be sent to the server + # We should find why we are missing remotes, as they will have been + # rejected. + + # Remove events from missing_remotes if they are referencing a missing + # remote. We only care about the "root" rejected ones. + missing_remote_ids = [e.event_id for e in missing_remotes] + base_remote_rejected = list(missing_remotes) + for e in missing_remotes: + for e_id, _ in e.auth_events: + if e_id in missing_remote_ids: + base_remote_rejected.remove(e) + + reason_map = {} + + for e in base_remote_rejected: + reason = yield self.store.get_rejection_reason(e.event_id) + if reason is None: + # FIXME: ERRR?! + raise RuntimeError("") + + reason_map[e.event_id] = reason + + if reason == RejectedReason.AUTH_ERROR: + pass + elif reason == RejectedReason.REPLACED: + # TODO: Get proof + pass + elif reason == RejectedReason.NOT_ANCESTOR: + # TODO: Get proof. + pass + + defer.returnValue({ + "rejects": { + e.event_id: { + "reason": reason_map[e.event_id], + "proof": None, + } + for e in base_remote_rejected + }, + "missing": missing_locals, + }) diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py index 7d38b31f44..b7249700d7 100644 --- a/synapse/storage/rejections.py +++ b/synapse/storage/rejections.py @@ -31,3 +31,13 @@ class RejectionsStore(SQLBaseStore): "last_failure": self._clock.time_msec(), } ) + + def get_rejection_reason(self, event_id): + self._simple_select_one_onecol( + table="rejections", + retcol="reason", + keyvalues={ + "event_id": event_id, + }, + allow_none=True, + ) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index bc7c6b6ed5..5866a387f6 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -128,5 +128,6 @@ CREATE TABLE IF NOT EXISTS rejections( event_id TEXT NOT NULL, reason TEXT NOT NULL, last_check TEXT NOT NULL, + root_rejected TEXT, CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE ); -- cgit 1.4.1 From c23e3db544eb940d95a092b661e3872480f3bf30 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 16:45:18 +0000 Subject: Add filter JSON sanity checks. --- synapse/api/filtering.py | 109 +++++++++++++++++++++++++++++++-- synapse/rest/client/v2_alpha/filter.py | 2 +- synapse/storage/filtering.py | 4 +- tests/api/test_filtering.py | 24 ++++++-- 4 files changed, 128 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 20b6951d47..6c7a73b6d5 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer +from synapse.api.errors import SynapseError +from synapse.types import UserID, RoomID class Filtering(object): @@ -25,10 +26,110 @@ class Filtering(object): def get_user_filter(self, user_localpart, filter_id): return self.store.get_user_filter(user_localpart, filter_id) - def add_user_filter(self, user_localpart, definition): - # TODO(paul): implement sanity checking of the definition - return self.store.add_user_filter(user_localpart, definition) + def add_user_filter(self, user_localpart, user_filter): + self._check_valid_filter(user_filter) + return self.store.add_user_filter(user_localpart, user_filter) # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for # them however + + def _check_valid_filter(self, user_filter): + """Check if the provided filter is valid. + + This inspects all definitions contained within the filter. + + Args: + user_filter(dict): The filter + Raises: + SynapseError: If the filter is not valid. + """ + # NB: Filters are the complete json blobs. "Definitions" are an + # individual top-level key e.g. public_user_data. Filters are made of + # many definitions. + + top_level_definitions = [ + "public_user_data", "private_user_data", "server_data" + ] + + room_level_definitions = [ + "state", "events", "ephemeral" + ] + + for key in top_level_definitions: + if key in user_filter: + self._check_definition(user_filter[key]) + + if "room" in user_filter: + for key in room_level_definitions: + if key in user_filter["room"]: + self._check_definition(user_filter["room"][key]) + + + def _check_definition(self, definition): + """Check if the provided definition is valid. + + This inspects not only the types but also the values to make sure they + make sense. + + Args: + definition(dict): The filter definition + Raises: + SynapseError: If there was a problem with this definition. + """ + # NB: Filters are the complete json blobs. "Definitions" are an + # individual top-level key e.g. public_user_data. Filters are made of + # many definitions. + if type(definition) != dict: + raise SynapseError( + 400, "Expected JSON object, not %s" % (definition,) + ) + + # check rooms are valid room IDs + room_id_keys = ["rooms", "not_rooms"] + for key in room_id_keys: + if key in definition: + if type(definition[key]) != list: + raise SynapseError(400, "Expected %s to be a list." % key) + for room_id in definition[key]: + RoomID.from_string(room_id) + + # check senders are valid user IDs + user_id_keys = ["senders", "not_senders"] + for key in user_id_keys: + if key in definition: + if type(definition[key]) != list: + raise SynapseError(400, "Expected %s to be a list." % key) + for user_id in definition[key]: + UserID.from_string(user_id) + + # TODO: We don't limit event type values but we probably should... + # check types are valid event types + event_keys = ["types", "not_types"] + for key in event_keys: + if key in definition: + if type(definition[key]) != list: + raise SynapseError(400, "Expected %s to be a list." % key) + for event_type in definition[key]: + if not isinstance(event_type, basestring): + raise SynapseError(400, "Event type should be a string") + + try: + event_format = definition["format"] + if event_format not in ["federation", "events"]: + raise SynapseError(400, "Invalid format: %s" % (event_format,)) + except KeyError: + pass # format is optional + + try: + event_select_list = definition["select"] + for select_key in event_select_list: + if select_key not in ["event_id", "origin_server_ts", + "thread_id", "content", "content.body"]: + raise SynapseError(400, "Bad select: %s" % (select_key,)) + except KeyError: + pass # select is optional + + if ("bundle_updates" in definition and + type(definition["bundle_updates"]) != bool): + raise SynapseError(400, "Bad bundle_updates: expected bool.") diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 09e44e8ae0..81a3e95155 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -93,7 +93,7 @@ class CreateFilterRestServlet(RestServlet): filter_id = yield self.filtering.add_user_filter( user_localpart=target_user.localpart, - definition=content, + user_filter=content, ) defer.returnValue((200, {"filter_id": str(filter_id)})) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index e98eaf8032..bab68a9eef 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -39,8 +39,8 @@ class FilteringStore(SQLBaseStore): defer.returnValue(json.loads(def_json)) - def add_user_filter(self, user_localpart, definition): - def_json = json.dumps(definition) + def add_user_filter(self, user_localpart, user_filter): + def_json = json.dumps(user_filter) # Need an atomic transaction to SELECT the maximal ID so far then # INSERT a new one diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 149948374d..188fbfb91e 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -57,13 +57,21 @@ class FilteringTestCase(unittest.TestCase): @defer.inlineCallbacks def test_add_filter(self): + user_filter = { + "room": { + "state": { + "types": ["m.*"] + } + } + } + filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, - definition={"type": ["m.*"]}, + user_filter=user_filter, ) self.assertEquals(filter_id, 0) - self.assertEquals({"type": ["m.*"]}, + self.assertEquals(user_filter, (yield self.datastore.get_user_filter( user_localpart=user_localpart, filter_id=0, @@ -72,9 +80,17 @@ class FilteringTestCase(unittest.TestCase): @defer.inlineCallbacks def test_get_filter(self): + user_filter = { + "room": { + "state": { + "types": ["m.*"] + } + } + } + filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - definition={"type": ["m.*"]}, + user_filter=user_filter, ) filter = yield self.filtering.get_user_filter( @@ -82,4 +98,4 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - self.assertEquals(filter, {"type": ["m.*"]}) + self.assertEquals(filter, user_filter) -- cgit 1.4.1 From 388581e087a3658c1b70d2aa1d17a132953350ca Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 16:58:23 +0000 Subject: Extract the id token of the token when authing users, include the token and device_id in the internal meta data for the event along with the transaction id when sending events --- synapse/api/auth.py | 8 ++-- synapse/handlers/message.py | 12 +++++- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/directory.py | 4 +- synapse/rest/client/v1/events.py | 4 +- synapse/rest/client/v1/initial_sync.py | 2 +- synapse/rest/client/v1/presence.py | 8 ++-- synapse/rest/client/v1/profile.py | 4 +- synapse/rest/client/v1/room.py | 64 +++++++++++++++++------------ synapse/rest/client/v1/voip.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/rest/media/v1/upload_resource.py | 2 +- synapse/storage/registration.py | 3 +- synapse/types.py | 3 ++ tests/rest/client/v1/test_presence.py | 2 + tests/rest/client/v1/test_rooms.py | 7 ++++ tests/rest/client/v1/test_typing.py | 1 + tests/storage/test_registration.py | 10 ++++- 18 files changed, 92 insertions(+), 48 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 292e9e2a80..3959e06a8b 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,7 +21,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from synapse.types import UserID +from synapse.types import UserID, ClientID import logging @@ -292,7 +292,7 @@ class Auth(object): Returns: Tuple of UserID and device string: User ID object of the user making the request - Device ID string of the device the user is using + Client ID object of the client instance the user is using Raises: AuthError if no user by that token exists or the token is invalid. """ @@ -302,6 +302,7 @@ class Auth(object): user_info = yield self.get_user_by_token(access_token) user = user_info["user"] device_id = user_info["device_id"] + token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders( @@ -317,7 +318,7 @@ class Auth(object): user_agent=user_agent ) - defer.returnValue((user, device_id)) + defer.returnValue((user, ClientID(device_id, token_id))) except KeyError: raise AuthError(403, "Missing access token.") @@ -342,6 +343,7 @@ class Auth(object): "admin": bool(ret.get("admin", False)), "device_id": ret.get("device_id"), "user": UserID.from_string(ret.get("name")), + "token_id": ret.get("token_id", None), } defer.returnValue(user_info) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9c3271fe88..6fbd2af4ab 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -114,7 +114,8 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def create_and_send_event(self, event_dict, ratelimit=True): + def create_and_send_event(self, event_dict, ratelimit=True, + client=None, txn_id=None): """ Given a dict from a client, create and handle a new event. Creates an FrozenEvent object, filling out auth_events, prev_events, @@ -148,6 +149,15 @@ class MessageHandler(BaseHandler): builder.content ) + if client is not None: + if client.token_id is not None: + builder.internal_metadata.token_id = client.token_id + if client.device_id is not None: + builder.internal_metadata.device_id = client.device_id + + if txn_id is not None: + builder.internal_metadata.txn_id = txn_id + event, context = yield self._create_new_client_event( builder=builder, ) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 6cfce1a479..2ce754b028 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -31,7 +31,7 @@ class WhoisRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) if not is_admin and target_user != auth_user: diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index ef853af411..8f65efec5f 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -45,7 +45,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) if not "room_id" in content: @@ -85,7 +85,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request, room_alias): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index e58ee46fcd..77b7c25a03 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -34,7 +34,7 @@ class EventStreamRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) try: handler = self.handlers.event_stream_handler pagin_config = PaginationConfig.from_request(request) @@ -71,7 +71,7 @@ class EventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, event_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 78d30abbf8..4a259bba64 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -25,7 +25,7 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 74669274a7..7feb4aadb1 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -32,7 +32,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( @@ -42,7 +42,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = {} @@ -77,7 +77,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): @@ -97,7 +97,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index f04abb2c26..15d6f3fc6c 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -37,7 +37,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: @@ -70,7 +70,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c8c34b4801..410f19ccf6 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -62,7 +62,7 @@ class RoomCreateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) room_config = self.get_room_config(request) info = yield self.make_room(room_config, auth_user, None) @@ -125,7 +125,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) msg_handler = self.handlers.message_handler data = yield msg_handler.get_room_data( @@ -142,8 +142,8 @@ class RoomStateEventRestServlet(ClientV1RestServlet): defer.returnValue((200, data.get_dict()["content"])) @defer.inlineCallbacks - def on_PUT(self, request, room_id, event_type, state_key): - user, device_id = yield self.auth.get_user_by_req(request) + def on_PUT(self, request, room_id, event_type, state_key, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -158,7 +158,9 @@ class RoomStateEventRestServlet(ClientV1RestServlet): event_dict["state_key"] = state_key msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event(event_dict) + yield msg_handler.create_and_send_event( + event_dict, client=client, txn_id=txn_id, + ) defer.returnValue((200, {})) @@ -172,8 +174,8 @@ class RoomSendEventRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server, with_get=True) @defer.inlineCallbacks - def on_POST(self, request, room_id, event_type): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, event_type, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -183,7 +185,9 @@ class RoomSendEventRestServlet(ClientV1RestServlet): "content": content, "room_id": room_id, "sender": user.to_string(), - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"event_id": event.event_id})) @@ -200,7 +204,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, event_type) + response = yield self.on_POST(request, room_id, event_type, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -215,8 +219,8 @@ class JoinRoomAliasServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_identifier): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_identifier, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) # the identifier could be a room alias or a room id. Try one then the # other if it fails to parse, without swallowing other valid @@ -245,7 +249,9 @@ class JoinRoomAliasServlet(ClientV1RestServlet): "room_id": identifier.to_string(), "sender": user.to_string(), "state_key": user.to_string(), - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"room_id": identifier.to_string()})) @@ -259,7 +265,7 @@ class JoinRoomAliasServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_identifier) + response = yield self.on_POST(request, room_identifier, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -283,7 +289,7 @@ class RoomMemberListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.room_member_handler members = yield handler.get_room_members_as_pagination_chunk( room_id=room_id, @@ -311,7 +317,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request( request, default_limit=10, ) @@ -335,7 +341,7 @@ class RoomStateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.message_handler # Get all the current state for this room events = yield handler.get_state_events( @@ -351,7 +357,7 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, @@ -395,8 +401,8 @@ class RoomMembershipRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_id, membership_action): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, membership_action, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -418,7 +424,9 @@ class RoomMembershipRestServlet(ClientV1RestServlet): "room_id": room_id, "sender": user.to_string(), "state_key": state_key, - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {})) @@ -432,7 +440,9 @@ class RoomMembershipRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, membership_action) + response = yield self.on_POST( + request, room_id, membership_action, txn_id + ) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -444,8 +454,8 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_id, event_id): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, event_id, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -456,7 +466,9 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): "room_id": room_id, "sender": user.to_string(), "redacts": event_id, - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"event_id": event.event_id})) @@ -470,7 +482,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, event_id) + response = yield self.on_POST(request, room_id, event_id, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -483,7 +495,7 @@ class RoomTypingRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) target_user = UserID.from_string(urllib.unquote(user_id)) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 42d8e30bab..11d08fbced 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -28,7 +28,7 @@ class VoipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) turnUris = self.hs.config.turn_uris turnSecret = self.hs.config.turn_shared_secret diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index 311ab89edb..22e26e3cd5 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -66,7 +66,7 @@ class ContentRepoResource(resource.Resource): @defer.inlineCallbacks def map_request_to_name(self, request): # auth the user - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) # namespace all file uploads on the user prefix = base64.urlsafe_b64encode( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 6bed8a8efa..b939a30e19 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -42,7 +42,7 @@ class UploadResource(BaseMediaResource): @defer.inlineCallbacks def _async_render_POST(self, request): try: - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 75dffa4db2..029b07cc66 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -122,7 +122,8 @@ class RegistrationStore(SQLBaseStore): def _query_for_auth(self, txn, token): sql = ( - "SELECT users.name, users.admin, access_tokens.device_id" + "SELECT users.name, users.admin," + " access_tokens.device_id, access_tokens.id as token_id" " FROM users" " INNER JOIN access_tokens on users.id = access_tokens.user_id" " WHERE token = ?" diff --git a/synapse/types.py b/synapse/types.py index faac729ff2..46dbab5374 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -119,3 +119,6 @@ class StreamToken( d = self._asdict() d[key] = new_value return StreamToken(**d) + + +ClientID = namedtuple("ClientID", ("device_id", "token_id")) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index a4f2abf213..f849120a3e 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -75,6 +75,7 @@ class PresenceStateTestCase(unittest.TestCase): "user": UserID.from_string(myid), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -165,6 +166,7 @@ class PresenceListTestCase(unittest.TestCase): "user": UserID.from_string(myid), "admin": False, "device_id": None, + "token_id": 1, } hs.handlers.room_member_handler = Mock( diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 76ed550b75..81ead10e76 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -70,6 +70,7 @@ class RoomPermissionsTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -466,6 +467,7 @@ class RoomsMemberListTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -555,6 +557,7 @@ class RoomsCreateTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -657,6 +660,7 @@ class RoomTopicTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -773,6 +777,7 @@ class RoomMemberStateTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -909,6 +914,7 @@ class RoomMessagesTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -1013,6 +1019,7 @@ class RoomInitialSyncTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index c89b37d004..c5d5b06da3 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -73,6 +73,7 @@ class RoomTypingTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 84bfde7568..6f8bea2f61 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -53,7 +53,10 @@ class RegistrationStoreTestCase(unittest.TestCase): ) self.assertEquals( - {"admin": 0, "device_id": None, "name": self.user_id}, + {"admin": 0, + "device_id": None, + "name": self.user_id, + "token_id": 1}, (yield self.store.get_user_by_token(self.tokens[0])) ) @@ -63,7 +66,10 @@ class RegistrationStoreTestCase(unittest.TestCase): yield self.store.add_access_token_to_user(self.user_id, self.tokens[1]) self.assertEquals( - {"admin": 0, "device_id": None, "name": self.user_id}, + {"admin": 0, + "device_id": None, + "name": self.user_id, + "token_id": 2}, (yield self.store.get_user_by_token(self.tokens[1])) ) -- cgit 1.4.1 From c18e551640994c8b2c509509bcf664748dd05724 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 17:08:53 +0000 Subject: Add a : to the doc string after the type of the return value --- synapse/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3959e06a8b..f08cb76159 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -290,7 +290,7 @@ class Auth(object): Args: request - An HTTP request with an access_token query parameter. Returns: - Tuple of UserID and device string: + tuple : of UserID and device string: User ID object of the user making the request Client ID object of the client instance the user is using Raises: -- cgit 1.4.1 From 3cca61e006d7e69b6643721c01ab7d81a8c2f373 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 17:16:12 +0000 Subject: Rename ClientID to ClientInfo since it is a pair of IDs rather than a single identifier --- synapse/api/auth.py | 4 ++-- synapse/types.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f08cb76159..9c03024512 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,7 +21,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from synapse.types import UserID, ClientID +from synapse.types import UserID, ClientInfo import logging @@ -318,7 +318,7 @@ class Auth(object): user_agent=user_agent ) - defer.returnValue((user, ClientID(device_id, token_id))) + defer.returnValue((user, ClientInfo(device_id, token_id))) except KeyError: raise AuthError(403, "Missing access token.") diff --git a/synapse/types.py b/synapse/types.py index 46dbab5374..f6a1b0bbcf 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -121,4 +121,4 @@ class StreamToken( return StreamToken(**d) -ClientID = namedtuple("ClientID", ("device_id", "token_id")) +ClientInfo = namedtuple("ClientInfo", ("device_id", "token_id")) -- cgit 1.4.1 From c81a19552f12a16591ae695fa1ef660f9e38730e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 17:32:41 +0000 Subject: Add ports back to demo/start.sh --- demo/start.sh | 2 +- synapse/rest/client/v2_alpha/sync.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/demo/start.sh b/demo/start.sh index a7502e7a8d..bb2248770d 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -16,7 +16,7 @@ if [ $# -eq 1 ]; then fi fi -for port in 8080; do +for port in 8080 8081 8082; do echo "Starting server on port $port... " https_port=$((port + 400)) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 0c17208cd3..a0ab9839a6 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -80,7 +80,7 @@ class SyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) timeout = self.parse_integer(request, "timeout", default=0) limit = self.parse_integer(request, "limit", required=True) -- cgit 1.4.1 From 11634017f47779d784325da5513517ad76b0dbc1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 17:42:19 +0000 Subject: s/definition/filter_json/ since definition is now used to mean a component of the filter, rather than the complete json --- synapse/storage/filtering.py | 4 ++-- synapse/storage/schema/filtering.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index bab68a9eef..cb01c2040f 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -33,7 +33,7 @@ class FilteringStore(SQLBaseStore): "user_id": user_localpart, "filter_id": filter_id, }, - retcol="definition", + retcol="filter_json", allow_none=False, ) @@ -57,7 +57,7 @@ class FilteringStore(SQLBaseStore): filter_id = max_id + 1 sql = ( - "INSERT INTO user_filters (user_id, filter_id, definition)" + "INSERT INTO user_filters (user_id, filter_id, filter_json)" "VALUES(?, ?, ?)" ) txn.execute(sql, (user_localpart, filter_id, def_json)) diff --git a/synapse/storage/schema/filtering.sql b/synapse/storage/schema/filtering.sql index 795aca4afd..beb39ca201 100644 --- a/synapse/storage/schema/filtering.sql +++ b/synapse/storage/schema/filtering.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id TEXT, filter_id INTEGER, - definition TEXT, + filter_json TEXT, FOREIGN KEY(user_id) REFERENCES users(id) ); -- cgit 1.4.1 From 8552ed8df2990d79b0015e0e84dd98de25fd0a9d Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 18:04:40 +0000 Subject: Change uses of get_user_by_req because it returns a tuple now. --- synapse/rest/client/v1/push_rule.py | 6 +++--- synapse/rest/client/v1/pusher.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2b33bdac08..64743a2f46 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -138,7 +138,7 @@ class PushRuleRestServlet(ClientV1RestServlet): except InvalidRuleException as e: raise SynapseError(400, e.message) - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -184,7 +184,7 @@ class PushRuleRestServlet(ClientV1RestServlet): except InvalidRuleException as e: raise SynapseError(400, e.message) - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) if 'device' in spec: rules = yield self.hs.get_datastore().get_push_rules_for_user_name( @@ -215,7 +215,7 @@ class PushRuleRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 80a11890a3..72d5e9e476 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -27,7 +27,7 @@ class PusherRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) content = _parse_json(request) -- cgit 1.4.1 From d5bdf3c0c7958e6a080f9ec4b38a51428717d02a Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 18:06:04 +0000 Subject: Allow the push rule delete method to take more specifiers. --- synapse/storage/push_rule.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 0342996ed1..c7b553292e 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -175,14 +175,17 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) @defer.inlineCallbacks - def delete_push_rule(self, user_name, rule_id): - yield self._simple_delete_one( - PushRuleTable.table_name, - { - 'user_name': user_name, - 'rule_id': rule_id - } - ) + def delete_push_rule(self, user_name, rule_id, **kwargs): + """ + Delete a push rule. Args specify the row to be deleted and can be + any of the columns in the push_rule table, but below are the + standard ones + + Args: + user_name (str): The matrix ID of the push rule owner + rule_id (str): The rule_id of the rule to be deleted + """ + yield self._simple_delete_one(PushRuleTable.table_name, kwargs) class RuleNotFoundException(Exception): -- cgit 1.4.1 From b0b80074e04dcb2b70dff56b0368060a19c065d3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 01:48:36 +0000 Subject: SYN-252: Supply the stream and topological parts in the correct order to the constructor --- synapse/storage/stream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8ac2adab05..062ca06fb3 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -82,10 +82,10 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): def parse(cls, string): try: if string[0] == 's': - return cls(None, int(string[1:])) + return cls(topological=None, stream=int(string[1:])) if string[0] == 't': parts = string[1:].split('-', 1) - return cls(int(parts[1]), int(parts[0])) + return cls(topological=int(parts[0]), stream=int(parts[1])) except: pass raise SynapseError(400, "Invalid token %r" % (string,)) @@ -94,7 +94,7 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): def parse_stream_token(cls, string): try: if string[0] == 's': - return cls(None, int(string[1:])) + return cls(topological=None, stream=int(string[1:])) except: pass raise SynapseError(400, "Invalid token %r" % (string,)) -- cgit 1.4.1 From 1b4a164c026ac027b7c84422f6c743c36896a0e7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 02:34:35 +0000 Subject: Add support for formatting events in the way a v2 client expects --- synapse/events/utils.py | 91 ++++++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 39 deletions(-) (limited to 'synapse') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 42fb0371e5..dd7f3d6f42 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,7 +89,46 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): +def format_event_raw(d): + return d + + +def format_event_for_client_v1(d): + d["user_id"] = d.pop("sender", None) + + move_keys = ("age", "redacted_because", "replaces_state", "prev_content") + for key in move_keys: + if key in d["unsigned"]: + d[key] = d["unsigned"][key] + + drop_keys = ( + "auth_events", "prev_events", "hashes", "signatures", "depth", + "unsigned", "origin" + ) + for key in drop_keys: + d.pop(key, None) + return d + + +def format_event_for_client_v2(d): + drop_keys = ( + "auth_events", "prev_events", "hashes", "signatures", "depth", "origin" + ) + for key in drop_keys: + d.pop(key, None) + return d + + +def format_event_for_client_v2_without_event_id(d): + d = format_event_for_client_v2(d) + d.pop("room_id", None) + d.pop("event_id", None) + return d + + +def serialize_event(e, time_now_ms, as_client_event=True, + event_format=format_event_for_client_v1, + token_id=None): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e @@ -99,48 +138,22 @@ def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): # Should this strip out None's? d = {k: v for k, v in e.get_dict().items()} - if not client_event: - # set the age and keep all other keys - if "age_ts" in d["unsigned"]: - d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] - return d - if "age_ts" in d["unsigned"]: - d["age"] = time_now_ms - d["unsigned"]["age_ts"] - del d["unsigned"]["age_ts"] - - d["user_id"] = d.pop("sender", None) + d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] + d["unsigned"]["age_ts"] if "redacted_because" in e.unsigned: - d["redacted_because"] = serialize_event( + d["unsigned"]["redacted_because"] = serialize_event( e.unsigned["redacted_because"], time_now_ms ) - del d["unsigned"]["redacted_because"] - - if "redacted_by" in e.unsigned: - d["redacted_by"] = e.unsigned["redacted_by"] - del d["unsigned"]["redacted_by"] + if token_id is not None: + if token_id == e.internal_metadata["token_id"]: + txn_id = e.internal_metadata.get("txn_id", None) + if txn_id is not None: + d["unsigned"]["transaction_id"] = txn_id - if "replaces_state" in e.unsigned: - d["replaces_state"] = e.unsigned["replaces_state"] - del d["unsigned"]["replaces_state"] - - if "prev_content" in e.unsigned: - d["prev_content"] = e.unsigned["prev_content"] - del d["unsigned"]["prev_content"] - - del d["auth_events"] - del d["prev_events"] - del d["hashes"] - del d["signatures"] - d.pop("depth", None) - d.pop("unsigned", None) - d.pop("origin", None) - d.pop("prev_state", None) - - if strip_ids: - d.pop("room_id", None) - d.pop("event_id", None) - - return d + if as_client_event: + return event_format(d) + else: + return d -- cgit 1.4.1 From b9c442c85c9f8c2aa7fcb57d6132dec9b85b4e60 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 02:45:33 +0000 Subject: Include transaction ids in unsigned section of events in the sync results for the clients that made those requests --- synapse/events/utils.py | 11 ++++++----- synapse/rest/client/v2_alpha/sync.py | 23 ++++++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) (limited to 'synapse') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index dd7f3d6f42..21316cc125 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -103,7 +103,7 @@ def format_event_for_client_v1(d): drop_keys = ( "auth_events", "prev_events", "hashes", "signatures", "depth", - "unsigned", "origin" + "unsigned", "origin", "prev_state" ) for key in drop_keys: d.pop(key, None) @@ -112,7 +112,8 @@ def format_event_for_client_v1(d): def format_event_for_client_v2(d): drop_keys = ( - "auth_events", "prev_events", "hashes", "signatures", "depth", "origin" + "auth_events", "prev_events", "hashes", "signatures", "depth", + "origin", "prev_state", ) for key in drop_keys: d.pop(key, None) @@ -140,7 +141,7 @@ def serialize_event(e, time_now_ms, as_client_event=True, if "age_ts" in d["unsigned"]: d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] - d["unsigned"]["age_ts"] + del d["unsigned"]["age_ts"] if "redacted_because" in e.unsigned: d["unsigned"]["redacted_because"] = serialize_event( @@ -148,8 +149,8 @@ def serialize_event(e, time_now_ms, as_client_event=True, ) if token_id is not None: - if token_id == e.internal_metadata["token_id"]: - txn_id = e.internal_metadata.get("txn_id", None) + if token_id == getattr(e.internal_metadata, "token_id", None): + txn_id = getattr(e.internal_metadata, "txn_id", None) if txn_id is not None: d["unsigned"]["transaction_id"] = txn_id diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index a0ab9839a6..4d950f9956 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -18,7 +18,9 @@ from twisted.internet import defer from synapse.http.servlet import RestServlet from synapse.handlers.sync import SyncConfig from synapse.types import StreamToken -from synapse.events.utils import serialize_event +from synapse.events.utils import ( + serialize_event, format_event_for_client_v2_without_event_id, +) from ._base import client_v2_pattern import logging @@ -139,7 +141,9 @@ class SyncRestServlet(RestServlet): "private_user_data": self.encode_events( sync_result.private_user_data, filter, time_now ), - "rooms": self.encode_rooms(sync_result.rooms, filter, time_now), + "rooms": self.encode_rooms( + sync_result.rooms, filter, time_now, client.token_id + ), "next_batch": sync_result.next_batch.to_string(), } @@ -153,25 +157,30 @@ class SyncRestServlet(RestServlet): # TODO(mjark): Respect formatting requirements in the filter. return serialize_event(event, time_now) - def encode_rooms(self, rooms, filter, time_now): - return [self.encode_room(room, filter, time_now) for room in rooms] + def encode_rooms(self, rooms, filter, time_now, token_id): + return [ + self.encode_room(room, filter, time_now, token_id) + for room in rooms + ] @staticmethod - def encode_room(room, filter, time_now): + def encode_room(room, filter, time_now, token_id): event_map = {} state_event_ids = [] recent_event_ids = [] for event in room.state: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( - event, time_now, strip_ids=True + event, time_now, token_id=token_id, + event_format=format_event_for_client_v2_without_event_id, ) state_event_ids.append(event.event_id) for event in room.events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( - event, time_now, strip_ids=True + event, time_now, token_id=token_id, + event_format=format_event_for_client_v2_without_event_id, ) recent_event_ids.append(event.event_id) return { -- cgit 1.4.1 From 3dbce6f4a59fde2a67e563ce338f510feda2dd1a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 03:33:51 +0000 Subject: Add typing notifications to sync --- synapse/handlers/sync.py | 30 +++++++++++++++++++++--------- synapse/rest/client/v2_alpha/sync.py | 18 ++++++++---------- 2 files changed, 29 insertions(+), 19 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 82a2c6986a..b86e783e14 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -44,11 +44,12 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "events", "state", "prev_batch", + "typing", ])): __slots__ = [] def __nonzero__(self): - return bool(self.events or self.state) + return bool(self.events or self.state or self.typing) class SyncResult(collections.namedtuple("SyncResult", [ @@ -196,15 +197,25 @@ class SyncHandler(BaseHandler): now_token = yield self.event_sources.get_current_token() - presence_stream = self.event_sources.sources["presence"] - pagination_config = PaginationConfig( - from_token=since_token, to_token=now_token + presence_source = self.event_sources.sources["presence"] + presence, presence_key = yield presence_source.get_new_events_for_user( + user=sync_config.user, + from_key=since_token.presence_key, + limit=sync_config.limit, ) - presence, _ = yield presence_stream.get_pagination_rows( + now_token = now_token.copy_and_replace("presence_key", presence_key) + + typing_source = self.event_sources.sources["typing"] + typing, typing_key = yield typing_source.get_new_events_for_user( user=sync_config.user, - pagination_config=pagination_config.get_source_config("presence"), - key=None + from_key=since_token.typing_key, + limit=sync_config.limit, ) + now_token = now_token.copy_and_replace("typing_key", typing_key) + + typing_by_room = {event["room_id"]: event for event in typing} + logger.debug("Typing %r", typing_by_room) + room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=sync_config.user.to_string(), membership_list=[Membership.INVITE, Membership.JOIN] @@ -218,7 +229,7 @@ class SyncHandler(BaseHandler): for event in room_list: room_sync = yield self.incremental_sync_with_gap_for_room( event.room_id, sync_config, since_token, now_token, - published_room_ids + published_room_ids, typing_by_room ) if room_sync: rooms.append(room_sync) @@ -233,7 +244,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, - published_room_ids): + published_room_ids, typing_by_room): """ Get the incremental delta needed to bring the client up to date for the room. Gives the client the most recent events and the changes to state. @@ -285,6 +296,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, + typing=typing_by_room.get(room_id, None) ) logging.debug("Room sync: %r", room_sync) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 4d950f9956..76489e27c8 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -135,10 +135,10 @@ class SyncRestServlet(RestServlet): time_now = self.clock.time_msec() response_content = { - "public_user_data": self.encode_events( + "public_user_data": self.encode_user_data( sync_result.public_user_data, filter, time_now ), - "private_user_data": self.encode_events( + "private_user_data": self.encode_user_data( sync_result.private_user_data, filter, time_now ), "rooms": self.encode_rooms( @@ -149,13 +149,8 @@ class SyncRestServlet(RestServlet): defer.returnValue((200, response_content)) - def encode_events(self, events, filter, time_now): - return [self.encode_event(event, filter, time_now) for event in events] - - @staticmethod - def encode_event(event, filter, time_now): - # TODO(mjark): Respect formatting requirements in the filter. - return serialize_event(event, time_now) + def encode_user_data(self, events, filter, time_now): + return events def encode_rooms(self, rooms, filter, time_now, token_id): return [ @@ -183,7 +178,7 @@ class SyncRestServlet(RestServlet): event_format=format_event_for_client_v2_without_event_id, ) recent_event_ids.append(event.event_id) - return { + result = { "room_id": room.room_id, "event_map": event_map, "events": { @@ -194,6 +189,9 @@ class SyncRestServlet(RestServlet): "limited": room.limited, "published": room.published, } + if room.typing is not None: + result["typing"] = room.typing + return result def register_servlets(hs, http_server): -- cgit 1.4.1 From e3e72b8c5c004c403929406fc952141fd0f1b8ae Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 03:35:25 +0000 Subject: Remove typing TODO --- synapse/handlers/sync.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b86e783e14..9e1188da56 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -255,7 +255,6 @@ class SyncHandler(BaseHandler): # the previous sync and this one. # TODO(mjark): Apply the event filter in sync_config # TODO(mjark): Check for redactions we might have missed. - # TODO(mjark): Typing notifications. recents, token = yield self.store.get_recent_events_for_room( room_id, limit=sync_config.limit + 1, -- cgit 1.4.1 From 3773759c0f1e25e6905e23368f770da99ceb3ea0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 09:15:33 +0000 Subject: Also edit the filter column on the delta SQL --- synapse/storage/schema/delta/v12.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql index 795aca4afd..beb39ca201 100644 --- a/synapse/storage/schema/delta/v12.sql +++ b/synapse/storage/schema/delta/v12.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id TEXT, filter_id INTEGER, - definition TEXT, + filter_json TEXT, FOREIGN KEY(user_id) REFERENCES users(id) ); -- cgit 1.4.1 From 2a4fda7b88cf91db8de2e524df162153d3f27094 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 09:27:16 +0000 Subject: Add filtering.filter_events function, with stub passes_filter function. --- synapse/api/filtering.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 6c7a73b6d5..d7ba6510ee 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -34,6 +34,21 @@ class Filtering(object): # replace_user_filter at some point? There's no REST API specified for # them however + def passes_filter(self, filter_json, event): + """Check if the event passes through the filter. + + Args: + filter_json(dict): The filter specification + event(Event): The event to check + Returns: + True if the event passes through the filter. + """ + return True + + def filter_events(self, events, user, filter_id): + filter_json = self.get_user_filter(user, filter_id) + return [e for e in events if self.passes_filter(filter_json, e)] + def _check_valid_filter(self, user_filter): """Check if the provided filter is valid. -- cgit 1.4.1 From 50de1eaad94715a1dda470f44f379683e5fa552b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 10:24:57 +0000 Subject: Add filtering public API; outline filtering algorithm. --- synapse/api/filtering.py | 60 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index d7ba6510ee..21fe72d6c2 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -30,25 +30,69 @@ class Filtering(object): self._check_valid_filter(user_filter) return self.store.add_user_filter(user_localpart, user_filter) + def filter_public_user_data(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["public_user_data"] + ) + + def filter_private_user_data(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["private_user_data"] + ) + + def filter_room_state(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["room", "state"] + ) + + def filter_room_events(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["room", "events"] + ) + + def filter_room_ephemeral(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["room", "ephemeral"] + ) + # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for # them however - def passes_filter(self, filter_json, event): - """Check if the event passes through the filter. + def _filter_on_key(self, events, user, filter_id, keys): + filter_json = self.get_user_filter(user.localpart, filter_id) + if not filter_json: + return events + + try: + # extract the right definition from the filter + definition = filter_json + for key in keys: + definition = definition[key] + return self._filter_with_definition(events, definition) + except KeyError: + return events # return all events if definition isn't specified. + + def _filter_with_definition(self, events, definition): + return [e for e in events if self._passes_definition(definition, e)] + + def _passes_definition(self, definition, event): + """Check if the event passes through the given definition. Args: - filter_json(dict): The filter specification - event(Event): The event to check + definition(dict): The definition to check against. + event(Event): The event to check. Returns: True if the event passes through the filter. """ + # Algorithm notes: + # For each key in the definition, check the event meets the criteria: + # * For types: Literal match or prefix match (if ends with wildcard) + # * For senders/rooms: Literal match only + # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' + # and 'not_types' then it is treated as only being in 'not_types') return True - def filter_events(self, events, user, filter_id): - filter_json = self.get_user_filter(user, filter_id) - return [e for e in events if self.passes_filter(filter_json, e)] - def _check_valid_filter(self, user_filter): """Check if the provided filter is valid. -- cgit 1.4.1 From 777d9914b537d06ebba91948a26d74d3a04b7284 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 11:38:06 +0000 Subject: Implement filter algorithm. Add basic event type unit tests to assert it works. --- synapse/api/filtering.py | 49 +++++++++++++++++++++++++++++++++++++++++++++ tests/api/test_filtering.py | 45 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 21fe72d6c2..8bc95aa394 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -91,8 +91,57 @@ class Filtering(object): # * For senders/rooms: Literal match only # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' # and 'not_types' then it is treated as only being in 'not_types') + + # room checks + if hasattr(event, "room_id"): + room_id = event.room_id + allow_rooms = definition["rooms"] if "rooms" in definition else None + reject_rooms = ( + definition["not_rooms"] if "not_rooms" in definition else None + ) + if reject_rooms and room_id in reject_rooms: + return False + if allow_rooms and room_id not in allow_rooms: + return False + + # sender checks + if hasattr(event, "sender"): + # Should we be including event.state_key for some event types? + sender = event.sender + allow_senders = ( + definition["senders"] if "senders" in definition else None + ) + reject_senders = ( + definition["not_senders"] if "not_senders" in definition else None + ) + if reject_senders and sender in reject_senders: + return False + if allow_senders and sender not in allow_senders: + return False + + # type checks + if "not_types" in definition: + for def_type in definition["not_types"]: + if self._event_matches_type(event, def_type): + return False + if "types" in definition: + included = False + for def_type in definition["types"]: + if self._event_matches_type(event, def_type): + included = True + break + if not included: + return False + return True + def _event_matches_type(self, event, def_type): + if def_type.endswith("*"): + type_prefix = def_type[:-1] + return event.type.startswith(type_prefix) + else: + return event.type == def_type + def _check_valid_filter(self, user_filter): """Check if the provided filter is valid. diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 188fbfb91e..4d40d88b00 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from collections import namedtuple from tests import unittest from twisted.internet import defer @@ -27,6 +27,7 @@ from synapse.server import HomeServer user_localpart = "test_user" +MockEvent = namedtuple("MockEvent", "sender type room_id") class FilteringTestCase(unittest.TestCase): @@ -55,6 +56,48 @@ class FilteringTestCase(unittest.TestCase): self.datastore = hs.get_datastore() + def test_definition_include_literal_types(self): + definition = { + "types": ["m.room.message", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!foo:bar" + ) + + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_include_wildcard_types(self): + definition = { + "types": ["m.*", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!foo:bar" + ) + + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_exclude_unknown_types(self): + definition = { + "types": ["m.room.message", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="now.for.something.completely.different", + room_id="!foo:bar" + ) + + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + @defer.inlineCallbacks def test_add_filter(self): user_filter = { -- cgit 1.4.1 From 83172487b05d7d99ccae0b353daee2f242445011 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 12:20:59 +0000 Subject: Add basic filtering public API unit tests. Use defers in the right places. --- synapse/api/filtering.py | 11 +++++---- tests/api/test_filtering.py | 54 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 8bc95aa394..7e239138b7 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID @@ -59,19 +60,21 @@ class Filtering(object): # replace_user_filter at some point? There's no REST API specified for # them however + @defer.inlineCallbacks def _filter_on_key(self, events, user, filter_id, keys): - filter_json = self.get_user_filter(user.localpart, filter_id) + filter_json = yield self.get_user_filter(user.localpart, filter_id) if not filter_json: - return events + defer.returnValue(events) try: # extract the right definition from the filter definition = filter_json for key in keys: definition = definition[key] - return self._filter_with_definition(events, definition) + defer.returnValue(self._filter_with_definition(events, definition)) except KeyError: - return events # return all events if definition isn't specified. + # return all events if definition isn't specified. + defer.returnValue(events) def _filter_with_definition(self, events, definition): return [e for e in events if self._passes_definition(definition, e)] diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 380dd97937..97fb9758e9 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -24,7 +24,7 @@ from tests.utils import ( ) from synapse.server import HomeServer - +from synapse.types import UserID user_localpart = "test_user" MockEvent = namedtuple("MockEvent", "sender type room_id") @@ -352,6 +352,58 @@ class FilteringTestCase(unittest.TestCase): self.filtering._passes_definition(definition, event) ) + @defer.inlineCallbacks + def test_filter_public_user_data_match(self): + user_filter = { + "public_user_data": { + "types": ["m.*"] + } + } + user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + user_filter=user_filter, + ) + event = MockEvent( + sender="@foo:bar", + type="m.profile", + room_id="!foo:bar" + ) + events = [event] + + results = yield self.filtering.filter_public_user_data( + events=events, + user=user, + filter_id=filter_id + ) + self.assertEquals(events, results) + + @defer.inlineCallbacks + def test_filter_public_user_data_no_match(self): + user_filter = { + "public_user_data": { + "types": ["m.*"] + } + } + user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + user_filter=user_filter, + ) + event = MockEvent( + sender="@foo:bar", + type="custom.avatar.3d.crazy", + room_id="!foo:bar" + ) + events = [event] + + results = yield self.filtering.filter_public_user_data( + events=events, + user=user, + filter_id=filter_id + ) + self.assertEquals([], results) + @defer.inlineCallbacks def test_add_filter(self): user_filter = { -- cgit 1.4.1 From c183cec8f61f5b3488973f01ca5203183a00e6d1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 13:44:52 +0000 Subject: Add post_json(...) method to federation client --- synapse/http/matrixfederationclient.py | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1dda3ba2c7..b1b2916fd3 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -244,6 +244,43 @@ class MatrixFederationHttpClient(object): defer.returnValue((response.code, body)) + @defer.inlineCallbacks + def post_json(self, destination, path, data={}): + """ Sends the specifed json data using POST + + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + data (dict): A dict containing the data that will be used as + the request body. This will be encoded as JSON. + + Returns: + Deferred: Succeeds when we get a 2xx HTTP response. The result + will be the decoded JSON body. On a 4xx or 5xx error response a + CodeMessageException is raised. + """ + + def body_callback(method, url_bytes, headers_dict): + self.sign_request( + destination, method, url_bytes, headers_dict, data + ) + return None + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + body_callback=body_callback, + headers_dict={"Content-Type": ["application/json"]}, + ) + + logger.debug("Getting resp body") + body = yield readBody(response) + logger.debug("Got resp body") + + defer.returnValue((response.code, body)) + @defer.inlineCallbacks def get_json(self, destination, path, args={}, retry_on_dns_fail=True): """ GETs some json from the given host homeserver and path -- cgit 1.4.1 From 5a3a15f5c186af6b818f2b2d3a4dafeee48b4e33 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 13:58:22 +0000 Subject: Make post_json(...) actually send data. --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b1b2916fd3..c7bf1b47b8 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -265,7 +265,7 @@ class MatrixFederationHttpClient(object): self.sign_request( destination, method, url_bytes, headers_dict, data ) - return None + return _JsonProducer(data) response = yield self._create_request( destination.encode("ascii"), -- cgit 1.4.1 From e016f4043b81ffdedf71c4459772f66757386e44 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 14:40:28 +0000 Subject: Use get_room_events_stream to get changes to the rooms if the number of changes is small --- synapse/handlers/sync.py | 56 +++++++++++++++++++++++++++++++++++++---------- synapse/storage/stream.py | 7 ++++++ 2 files changed, 52 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9e1188da56..e93dfe005d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -216,23 +216,57 @@ class SyncHandler(BaseHandler): typing_by_room = {event["room_id"]: event for event in typing} logger.debug("Typing %r", typing_by_room) - room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=sync_config.user.to_string(), - membership_list=[Membership.INVITE, Membership.JOIN] - ) + rm_handler = self.hs.get_handlers().room_member_handler + room_ids = yield rm_handler.get_rooms_for_user(sync_config.user) # TODO (mjark): Does public mean "published"? published_rooms = yield self.store.get_rooms(is_public=True) published_room_ids = set(r["room_id"] for r in published_rooms) + room_events, _ = yield self.store.get_room_events_stream( + sync_config.user.to_string(), + from_key=since_token.room_key, + to_key=now_token.room_key, + room_id=None, + limit=sync_config.limit + 1, + ) + rooms = [] - for event in room_list: - room_sync = yield self.incremental_sync_with_gap_for_room( - event.room_id, sync_config, since_token, now_token, - published_room_ids, typing_by_room - ) - if room_sync: - rooms.append(room_sync) + if len(room_events) <= sync_config.limit: + # There is no gap in any of the rooms. Therefore we can just + # partition the new events by room and return them. + events_by_room_id = {} + for event in room_events: + events_by_room_id.setdefault(event.room_id, []).append(event) + + for room_id in room_ids: + recents = events_by_room_id.get(room_id, []) + state = [event for event in recents if event.is_state()] + if recents: + prev_batch = now_token.copy_and_replace( + "room_key", recents[0].internal_metadata.before + ) + else: + prev_batch = now_token + room_sync = RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recents, + prev_batch=prev_batch, + state=state, + limited=False, + typing=typing_by_room.get(room_id, None) + ) + if room_sync is not None: + rooms.append(room_sync) + else: + for room_id in room_ids: + room_sync = yield self.incremental_sync_with_gap_for_room( + room_id, sync_config, since_token, now_token, + published_room_ids, typing_by_room + ) + if room_sync: + rooms.append(room_sync) defer.returnValue(SyncResult( public_user_data=presence, diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index db1816ea84..93ccfd8c10 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -181,6 +181,13 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + for event, row in zip(ret, rows): + stream = row["stream_ordering"] + topo = event.depth + internal = event.internal_metadata + internal.before = str(_StreamToken(topo, stream - 1)) + internal.after = str(_StreamToken(topo, stream)) + if rows: key = "s%d" % max([r["stream_ordering"] for r in rows]) else: -- cgit 1.4.1 From e4f50fa0aa3426a272b1526072c4c42802989ba4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 14:53:18 +0000 Subject: Move bump schema delta --- synapse/storage/schema/delta/v12.sql | 24 ------------------------ synapse/storage/schema/delta/v13.sql | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 24 deletions(-) delete mode 100644 synapse/storage/schema/delta/v12.sql create mode 100644 synapse/storage/schema/delta/v13.sql (limited to 'synapse') diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql deleted file mode 100644 index beb39ca201..0000000000 --- a/synapse/storage/schema/delta/v12.sql +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS user_filters( - user_id TEXT, - filter_id INTEGER, - filter_json TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) -); - -CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( - user_id, filter_id -); diff --git a/synapse/storage/schema/delta/v13.sql b/synapse/storage/schema/delta/v13.sql new file mode 100644 index 0000000000..beb39ca201 --- /dev/null +++ b/synapse/storage/schema/delta/v13.sql @@ -0,0 +1,24 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS user_filters( + user_id TEXT, + filter_id INTEGER, + filter_json TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); -- cgit 1.4.1 From 33391db5f8d9d0d365607ca50ba59ce72c90cda0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 15:54:54 +0000 Subject: Merge in auth changes from develop --- synapse/rest/client/v2_alpha/filter.py | 4 ++-- tests/rest/client/v2_alpha/__init__.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 81a3e95155..cee06ccaca 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -40,7 +40,7 @@ class GetFilterRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id, filter_id): target_user = UserID.from_string(user_id) - auth_user = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) if target_user != auth_user: raise AuthError(403, "Cannot get filters for other users") @@ -76,7 +76,7 @@ class CreateFilterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) if target_user != auth_user: raise AuthError(403, "Cannot create filters for other users") diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index 3fe62d5ac6..fa70575c57 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -51,6 +51,7 @@ class V2AlphaRestTestCase(unittest.TestCase): "user": UserID.from_string(self.USER_ID), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token -- cgit 1.4.1 From 9150a0d62ed4195b41834cea8a836332e74fb96b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:01:14 +0000 Subject: Fix code-style --- synapse/api/filtering.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 7e239138b7..e16c0e559f 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -74,7 +74,7 @@ class Filtering(object): defer.returnValue(self._filter_with_definition(events, definition)) except KeyError: # return all events if definition isn't specified. - defer.returnValue(events) + defer.returnValue(events) def _filter_with_definition(self, events, definition): return [e for e in events if self._passes_definition(definition, e)] @@ -94,14 +94,12 @@ class Filtering(object): # * For senders/rooms: Literal match only # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' # and 'not_types' then it is treated as only being in 'not_types') - + # room checks if hasattr(event, "room_id"): room_id = event.room_id - allow_rooms = definition["rooms"] if "rooms" in definition else None - reject_rooms = ( - definition["not_rooms"] if "not_rooms" in definition else None - ) + allow_rooms = definition.get("rooms", None) + reject_rooms = definition.get("not_rooms", None) if reject_rooms and room_id in reject_rooms: return False if allow_rooms and room_id not in allow_rooms: @@ -111,12 +109,8 @@ class Filtering(object): if hasattr(event, "sender"): # Should we be including event.state_key for some event types? sender = event.sender - allow_senders = ( - definition["senders"] if "senders" in definition else None - ) - reject_senders = ( - definition["not_senders"] if "not_senders" in definition else None - ) + allow_senders = definition.get("senders", None) + reject_senders = definition.get("not_senders", None) if reject_senders and sender in reject_senders: return False if allow_senders and sender not in allow_senders: @@ -176,7 +170,6 @@ class Filtering(object): if key in user_filter["room"]: self._check_definition(user_filter["room"][key]) - def _check_definition(self, definition): """Check if the provided definition is valid. -- cgit 1.4.1 From 8b1dd9f57f8afb2d602d3b533ab89dbe1df6b465 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 16:10:01 +0000 Subject: Only send a badge-reset if the user actually has unread notifications. --- synapse/push/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 10ac890482..fa967c5a5d 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -56,6 +56,7 @@ class Pusher(object): # The last value of last_active_time that we saw self.last_last_active_time = 0 + self.has_unread = True @defer.inlineCallbacks def _actions_for_event(self, ev): @@ -180,6 +181,7 @@ class Pusher(object): processed = True else: rejected = yield self.dispatch_push(single_event, tweaks) + self.has_unread = True if isinstance(rejected, list) or isinstance(rejected, tuple): processed = True for pk in rejected: @@ -290,9 +292,12 @@ class Pusher(object): if 'last_active' in state.state: last_active = state.state['last_active'] if last_active > self.last_last_active_time: - logger.info("Resetting badge count for %s", self.user_name) - self.reset_badge_count() self.last_last_active_time = last_active + if self.has_unread: + logger.info("Resetting badge count for %s", self.user_name) + self.reset_badge_count() + self.has_unread = False + def _value_for_dotted_key(dotted_key, event): -- cgit 1.4.1 From acb68a39e02f405c116135400e33a3b1940a07f8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:10:35 +0000 Subject: Code style fixes. --- synapse/api/errors.py | 1 + synapse/push/__init__.py | 15 +++++++-------- synapse/push/httppusher.py | 8 ++++---- synapse/push/pusherpool.py | 2 +- synapse/rest/__init__.py | 2 +- synapse/rest/client/v1/push_rule.py | 29 ++++++++++++++++++++++------- synapse/storage/push_rule.py | 9 +++++---- synapse/storage/pusher.py | 2 +- 8 files changed, 42 insertions(+), 26 deletions(-) (limited to 'synapse') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 5872e82d0f..ad478aa6b7 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -111,6 +111,7 @@ class NotFoundError(SynapseError): **kwargs ) + class AuthError(SynapseError): """An error raised when there was a problem authorising an event.""" diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index fa967c5a5d..472ede5480 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -189,8 +189,8 @@ class Pusher(object): # for sanity, we only remove the pushkey if it # was the one we actually sent... logger.warn( - ("Ignoring rejected pushkey %s because we " - "didn't send it"), pk + ("Ignoring rejected pushkey %s because we" + " didn't send it"), pk ) else: logger.info( @@ -236,8 +236,7 @@ class Pusher(object): # of old notifications. logger.warn("Giving up on a notification to user %s, " "pushkey %s", - self.user_name, self.pushkey - ) + self.user_name, self.pushkey) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token( @@ -258,8 +257,7 @@ class Pusher(object): "Trying again in %dms", self.user_name, self.clock.time_msec() - self.failing_since, - self.backoff_delay - ) + self.backoff_delay) yield synapse.util.async.sleep(self.backoff_delay / 1000.0) self.backoff_delay *= 2 if self.backoff_delay > Pusher.MAX_BACKOFF: @@ -299,7 +297,6 @@ class Pusher(object): self.has_unread = False - def _value_for_dotted_key(dotted_key, event): parts = dotted_key.split(".") val = event @@ -310,6 +307,7 @@ def _value_for_dotted_key(dotted_key, event): parts = parts[1:] return val + def _tweaks_for_actions(actions): tweaks = {} for a in actions: @@ -319,6 +317,7 @@ def _tweaks_for_actions(actions): tweaks['sound'] = a['set_sound'] return tweaks + class PusherConfigException(Exception): def __init__(self, msg): - super(PusherConfigException, self).__init__(msg) \ No newline at end of file + super(PusherConfigException, self).__init__(msg) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index e12b946727..ab128e31e5 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -71,11 +71,11 @@ class HttpPusher(Pusher): # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? #'from_display_name': 'Steve Stevington' - 'counts': { #-- we don't mark messages as read yet so - # we have no way of knowing + 'counts': { # -- we don't mark messages as read yet so + # we have no way of knowing # Just set the badge to 1 until we have read receipts 'unread': 1, - # 'missed_calls': 2 + # 'missed_calls': 2 }, 'devices': [ { @@ -142,4 +142,4 @@ class HttpPusher(Pusher): rejected = [] if 'rejected' in resp: rejected = resp['rejected'] - defer.returnValue(rejected) \ No newline at end of file + defer.returnValue(rejected) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 856defedac..4892c21e7b 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -149,4 +149,4 @@ class PusherPool: logger.info("Stopping pusher %s", fullid) self.pushers[fullid].stop() del self.pushers[fullid] - yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) \ No newline at end of file + yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 90afd93333..1a84d94cd9 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -11,4 +11,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 64743a2f46..2b1e930326 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -30,9 +30,9 @@ class PushRuleRestServlet(ClientV1RestServlet): 'sender': 1, 'room': 2, 'content': 3, - 'override': 4 + 'override': 4, } - PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} + PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( "Unrecognised request: You probably wanted a trailing slash") @@ -260,7 +260,9 @@ class PushRuleRestServlet(ClientV1RestServlet): if path == []: # we're a reference impl: pedantry is our job. - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': defer.returnValue((200, rules)) @@ -271,7 +273,9 @@ class PushRuleRestServlet(ClientV1RestServlet): elif path[0] == 'device': path = path[1:] if path == []: - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': defer.returnValue((200, rules['device'])) @@ -290,11 +294,13 @@ class PushRuleRestServlet(ClientV1RestServlet): def on_OPTIONS(self, _): return 200, {} + def _add_empty_priority_class_arrays(d): for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): d[pc] = [] return d + def _instance_handle_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the @@ -305,9 +311,12 @@ def _instance_handle_from_conditions(conditions): return c['instance_handle'] return None + def _filter_ruleset_with_path(ruleset, path): if path == []: - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': return ruleset @@ -316,7 +325,9 @@ def _filter_ruleset_with_path(ruleset, path): raise UnrecognizedRequestError() path = path[1:] if path == []: - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': return ruleset[template_kind] rule_id = path[0] @@ -325,6 +336,7 @@ def _filter_ruleset_with_path(ruleset, path): return r raise NotFoundError + def _priority_class_from_spec(spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) @@ -335,6 +347,7 @@ def _priority_class_from_spec(spec): return pc + def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device @@ -343,6 +356,7 @@ def _priority_class_to_template_name(pc): else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: @@ -359,8 +373,9 @@ def _rule_to_template(rule): ret["pattern"] = thecond["pattern"] return ret + def _strip_device_condition(rule): - for i,c in enumerate(rule['conditions']): + for i, c in enumerate(rule['conditions']): if c['kind'] == 'device': del rule['conditions'][i] return rule diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index c7b553292e..27502d2399 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -117,7 +117,7 @@ class PushRuleStore(SQLBaseStore): new_rule['priority'] = new_rule_priority sql = ( - "SELECT COUNT(*) FROM "+PushRuleTable.table_name+ + "SELECT COUNT(*) FROM " + PushRuleTable.table_name + " WHERE user_name = ? AND priority_class = ? AND priority = ?" ) txn.execute(sql, (user_name, priority_class, new_rule_priority)) @@ -146,10 +146,11 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) - def _add_push_rule_highest_priority_txn(self, txn, user_name, priority_class, **kwargs): + def _add_push_rule_highest_priority_txn(self, txn, user_name, + priority_class, **kwargs): # find the highest priority rule in that class sql = ( - "SELECT COUNT(*), MAX(priority) FROM "+PushRuleTable.table_name+ + "SELECT COUNT(*), MAX(priority) FROM " + PushRuleTable.table_name + " WHERE user_name = ? and priority_class = ?" ) txn.execute(sql, (user_name, priority_class)) @@ -209,4 +210,4 @@ class PushRuleTable(Table): "actions", ] - EntryType = collections.namedtuple("PushRuleEntry", fields) \ No newline at end of file + EntryType = collections.namedtuple("PushRuleEntry", fields) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 113cdc8a8e..f253c9e2c3 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -170,4 +170,4 @@ class PushersTable(Table): "failing_since" ] - EntryType = collections.namedtuple("PusherEntry", fields) \ No newline at end of file + EntryType = collections.namedtuple("PusherEntry", fields) -- cgit 1.4.1 From 4d9dd9bdc02ca7b524c90ae527687e7a838acee4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:23:03 +0000 Subject: Fix v2 initial sync --- synapse/handlers/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index e93dfe005d..cfe2944917 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -157,7 +157,7 @@ class SyncHandler(BaseHandler): )) @defer.inlineCallbacks - def intial_sync_for_room(self, room_id, sync_config, now_token, + def initial_sync_for_room(self, room_id, sync_config, now_token, published_room_ids): """Sync a room for a client which is starting without any state Returns: @@ -180,6 +180,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=current_state_events, limited=True, + typing=None, )) @defer.inlineCallbacks -- cgit 1.4.1 From cc42d3f9076db1815a989bab7ef6004796b31b3c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:27:38 +0000 Subject: Fix check for empty room update --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index cfe2944917..ec8beb4c6a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -258,7 +258,7 @@ class SyncHandler(BaseHandler): limited=False, typing=typing_by_room.get(room_id, None) ) - if room_sync is not None: + if room_sync: rooms.append(room_sync) else: for room_id in room_ids: -- cgit 1.4.1 From 722b65f46131349c5afdcc7eb48297cdd9d9cbd6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:41:21 +0000 Subject: Move typing notifs to an "emphermal" event list on the room object --- synapse/handlers/sync.py | 12 +++++++----- synapse/rest/client/v2_alpha/sync.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ec8beb4c6a..3860c3c95b 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -44,12 +44,12 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "events", "state", "prev_batch", - "typing", + "ephemeral", ])): __slots__ = [] def __nonzero__(self): - return bool(self.events or self.state or self.typing) + return bool(self.events or self.state or self.ephemeral) class SyncResult(collections.namedtuple("SyncResult", [ @@ -180,7 +180,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=current_state_events, limited=True, - typing=None, + ephemeral=[], )) @defer.inlineCallbacks @@ -214,7 +214,9 @@ class SyncHandler(BaseHandler): ) now_token = now_token.copy_and_replace("typing_key", typing_key) - typing_by_room = {event["room_id"]: event for event in typing} + typing_by_room = {event["room_id"]: [event] for event in typing} + for event in typing: + event.pop("room_id") logger.debug("Typing %r", typing_by_room) rm_handler = self.hs.get_handlers().room_member_handler @@ -256,7 +258,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch, state=state, limited=False, - typing=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, []) ) if room_sync: rooms.append(room_sync) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 76489e27c8..2ae2eec55d 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -66,6 +66,7 @@ class SyncRestServlet(RestServlet): } "state": [] // list of EventIDs updating the current state to // be what it should be at the end of the batch. + "ephemeral": [] }] } """ @@ -188,9 +189,8 @@ class SyncRestServlet(RestServlet): "state": state_event_ids, "limited": room.limited, "published": room.published, + "ephemeral": room.ephemeral, } - if room.typing is not None: - result["typing"] = room.typing return result -- cgit 1.4.1 From 4ad45f258245344218c5ac9ffcf20052bfdb5ba2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:41:49 +0000 Subject: Fix indent --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3860c3c95b..8c7681a48d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -158,7 +158,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def initial_sync_for_room(self, room_id, sync_config, now_token, - published_room_ids): + published_room_ids): """Sync a room for a client which is starting without any state Returns: A Deferred RoomSyncResult. -- cgit 1.4.1 From 78015948a7febb18e000651f72f8f58830a55b93 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 16:50:23 +0000 Subject: Initial implementation of auth conflict resolution --- synapse/events/utils.py | 6 +- synapse/federation/federation_client.py | 2 +- synapse/federation/federation_server.py | 33 +++++ synapse/federation/transport/client.py | 16 +++ synapse/federation/transport/server.py | 21 +++- synapse/handlers/federation.py | 207 ++++++++++++++++++++------------ synapse/storage/rejections.py | 4 +- tests/handlers/test_federation.py | 2 + 8 files changed, 210 insertions(+), 81 deletions(-) (limited to 'synapse') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index bcb5457278..10a6b9f264 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -45,12 +45,14 @@ def prune_event(event): "membership", ] + event_dict = event.get_dict() + new_content = {} def add_fields(*fields): for field in fields: if field in event.content: - new_content[field] = event.content[field] + new_content[field] = event_dict["content"][field] if event_type == EventTypes.Member: add_fields("membership") @@ -75,7 +77,7 @@ def prune_event(event): allowed_fields = { k: v - for k, v in event.get_dict().items() + for k, v in event_dict.items() if k in allowed_keys } diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index ebcd593506..1173ca817b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -345,7 +345,7 @@ class FederationClient(object): "auth_chain": [e.get_pdu_json(time_now) for e in local_auth], } - code, content = yield self.transport_layer.send_invite( + code, content = yield self.transport_layer.send_query_auth( destination=destination, room_id=room_id, event_id=event_id, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index fc5342afaa..8cff4e6472 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -230,6 +230,39 @@ class FederationServer(object): "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus], })) + @defer.inlineCallbacks + def on_query_auth_request(self, origin, content, event_id): + auth_chain = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content["auth_chain"] + ] + + missing = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content.get("missing", []) + ] + + ret = yield self.handler.on_query_auth( + origin, event_id, auth_chain, content.get("rejects", []), missing + ) + + time_now = self._clock.time_msec() + send_content = { + "auth_chain": [ + e.get_pdu_json(time_now) + for e in ret["auth_chain"] + ], + "rejects": content.get("rejects", []), + "missing": [ + e.get_pdu_json(time_now) + for e in ret.get("missing", []) + ], + } + + defer.returnValue( + (200, send_content) + ) + @log_function def _get_persisted_pdu(self, origin, event_id, do_auth=True): """ Get a PDU from the database with given origin and id. diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index e634a3a213..4cb1dea2de 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -213,3 +213,19 @@ class TransportLayerClient(object): ) defer.returnValue(response) + + @defer.inlineCallbacks + @log_function + def send_query_auth(self, destination, room_id, event_id, content): + path = PREFIX + "/query_auth/%s/%s" % (room_id, event_id) + + code, content = yield self.client.post_json( + destination=destination, + path=path, + data=content, + ) + + if not 200 <= code < 300: + raise RuntimeError("Got %d from send_invite", code) + + defer.returnValue(json.loads(content)) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index a380a6910b..9c9f8d525b 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -42,7 +42,7 @@ class TransportLayerServer(object): content = None origin = None - if request.method == "PUT": + if request.method in ["PUT", "POST"]: # TODO: Handle other method types? other content types? try: content_bytes = request.content.read() @@ -234,6 +234,16 @@ class TransportLayerServer(object): ) ) ) + self.server.register_path( + "POST", + re.compile("^" + PREFIX + "/query_auth/([^/]*)/([^/]*)$"), + self._with_authentication( + lambda origin, content, query, context, event_id: + self._on_query_auth_request( + origin, content, event_id, + ) + ) + ) @defer.inlineCallbacks @log_function @@ -325,3 +335,12 @@ class TransportLayerServer(object): ) defer.returnValue((200, content)) + + @defer.inlineCallbacks + @log_function + def _on_query_auth_request(self, origin, content, event_id): + new_content = yield self.request_handler.on_query_auth_request( + origin, content, event_id + ) + + defer.returnValue((200, new_content)) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 97e3c503b9..14c26d8cea 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -126,7 +126,7 @@ class FederationHandler(BaseHandler): if not state: state, auth_chain = yield replication.get_state_for_room( - origin, context=event.room_id, event_id=event.event_id, + origin, room_id=event.room_id, event_id=event.event_id, ) if not auth_chain: @@ -139,7 +139,7 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e, fetch_auth_from=origin) + yield self._handle_new_event(origin, e) except: logger.exception( "Failed to handle auth event %s", @@ -152,7 +152,7 @@ class FederationHandler(BaseHandler): for e in state: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e) + yield self._handle_new_event(origin, e) except: logger.exception( "Failed to handle state event %s", @@ -161,6 +161,7 @@ class FederationHandler(BaseHandler): try: yield self._handle_new_event( + origin, event, state=state, backfilled=backfilled, @@ -363,7 +364,14 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e) + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + target_host, e, auth_events=auth + ) except: logger.exception( "Failed to handle auth event %s", @@ -374,8 +382,13 @@ class FederationHandler(BaseHandler): # FIXME: Auth these. e.internal_metadata.outlier = True try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } yield self._handle_new_event( - e, fetch_auth_from=target_host + target_host, e, auth_events=auth ) except: logger.exception( @@ -384,6 +397,7 @@ class FederationHandler(BaseHandler): ) yield self._handle_new_event( + target_host, new_event, state=state, current_state=state, @@ -450,7 +464,7 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False - context = yield self._handle_new_event(event) + context = yield self._handle_new_event(origin, event) logger.debug( "on_send_join_request: After _handle_new_event: %s, sigs: %s", @@ -651,11 +665,12 @@ class FederationHandler(BaseHandler): waiters.pop().callback(None) @defer.inlineCallbacks - def _handle_new_event(self, event, state=None, backfilled=False, - current_state=None, fetch_auth_from=None): + @log_function + def _handle_new_event(self, origin, event, state=None, backfilled=False, + current_state=None, auth_events=None): logger.debug( - "_handle_new_event: Before annotate: %s, sigs: %s", + "_handle_new_event: %s, sigs: %s", event.event_id, event.signatures, ) @@ -663,62 +678,34 @@ class FederationHandler(BaseHandler): event, old_state=state ) + if not auth_events: + auth_events = context.auth_events + logger.debug( - "_handle_new_event: Before auth fetch: %s, sigs: %s", - event.event_id, event.signatures, + "_handle_new_event: %s, auth_events: %s", + event.event_id, auth_events, ) is_new_state = not event.internal_metadata.is_outlier() - known_ids = set( - [s.event_id for s in context.auth_events.values()] - ) - - for e_id, _ in event.auth_events: - if e_id not in known_ids: - e = yield self.store.get_event(e_id, allow_none=True) - - if not e and fetch_auth_from is not None: - # Grab the auth_chain over federation if we are missing - # auth events. - auth_chain = yield self.replication_layer.get_event_auth( - fetch_auth_from, event.event_id, event.room_id - ) - for auth_event in auth_chain: - yield self._handle_new_event(auth_event) - e = yield self.store.get_event(e_id, allow_none=True) - - if not e: - # TODO: Do some conflict res to make sure that we're - # not the ones who are wrong. - logger.info( - "Rejecting %s as %s not in db or %s", - event.event_id, e_id, known_ids, - ) - # FIXME: How does raising AuthError work with federation? - raise AuthError(403, "Cannot find auth event") - - context.auth_events[(e.type, e.state_key)] = e - - logger.debug( - "_handle_new_event: Before hack: %s, sigs: %s", - event.event_id, event.signatures, - ) - + # This is a hack to fix some old rooms where the initial join event + # didn't reference the create event in its auth events. if event.type == EventTypes.Member and not event.auth_events: if len(event.prev_events) == 1: c = yield self.store.get_event(event.prev_events[0][0]) if c.type == EventTypes.Create: - context.auth_events[(c.type, c.state_key)] = c - - logger.debug( - "_handle_new_event: Before auth check: %s, sigs: %s", - event.event_id, event.signatures, - ) + auth_events[(c.type, c.state_key)] = c try: - self.auth.check(event, auth_events=context.auth_events) - except AuthError: + yield self.do_auth( + origin, event, context, auth_events=auth_events + ) + except AuthError as e: + logger.warn( + "Rejecting %s because %s", + event.event_id, e.msg + ) + # TODO: Store rejection. context.rejected = RejectedReason.AUTH_ERROR @@ -731,11 +718,6 @@ class FederationHandler(BaseHandler): ) raise - logger.debug( - "_handle_new_event: Before persist_event: %s, sigs: %s", - event.event_id, event.signatures, - ) - yield self.store.persist_event( event, context=context, @@ -744,25 +726,73 @@ class FederationHandler(BaseHandler): current_state=current_state, ) - logger.debug( - "_handle_new_event: After persist_event: %s, sigs: %s", - event.event_id, event.signatures, + defer.returnValue(context) + + @defer.inlineCallbacks + def on_query_auth(self, origin, event_id, remote_auth_chain, rejects, + missing): + # Just go through and process each event in `remote_auth_chain`. We + # don't want to fall into the trap of `missing` being wrong. + for e in remote_auth_chain: + try: + yield self._handle_new_event(origin, e) + except AuthError: + pass + + # Now get the current auth_chain for the event. + local_auth_chain = yield self.store.get_auth_chain([event_id]) + + # TODO: Check if we would now reject event_id. If so we need to tell + # everyone. + + ret = yield self.construct_auth_difference( + local_auth_chain, remote_auth_chain ) - defer.returnValue(context) + logger.debug("on_query_auth reutrning: %s", ret) + + defer.returnValue(ret) @defer.inlineCallbacks - def do_auth(self, origin, event, context): - for e_id, _ in event.auth_events: - pass + @log_function + def do_auth(self, origin, event, context, auth_events): + # Check if we have all the auth events. + res = yield self.store.have_events( + [e_id for e_id, _ in event.auth_events] + ) - auth_events = set(e_id for e_id, _ in event.auth_events) - current_state = set(e.event_id for e in context.auth_events.values()) + event_auth_events = set(e_id for e_id, _ in event.auth_events) + seen_events = set(res.keys()) - missing_auth = auth_events - current_state + missing_auth = event_auth_events - seen_events if missing_auth: + logger.debug("Missing auth: %s", missing_auth) + # If we don't have all the auth events, we need to get them. + remote_auth_chain = yield self.replication_layer.get_event_auth( + origin, event.room_id, event.event_id + ) + + for e in remote_auth_chain: + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in remote_auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass + + current_state = set(e.event_id for e in auth_events.values()) + different_auth = event_auth_events - current_state + + if different_auth and not event.internal_metadata.is_outlier(): # Do auth conflict res. + logger.debug("Different auth: %s", different_auth) # 1. Get what we think is the auth chain. auth_ids = self.auth.compute_auth_events(event, context) @@ -778,14 +808,24 @@ class FederationHandler(BaseHandler): # 3. Process any remote auth chain events we haven't seen. for e in result.get("missing", []): - # TODO. - pass + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in result["auth_chain"] + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass # 4. Look at rejects and their proofs. # TODO. try: - self.auth.check(event, auth_events=context.auth_events) + self.auth.check(event, auth_events=auth_events) except AuthError: raise @@ -802,12 +842,16 @@ class FederationHandler(BaseHandler): dict """ + logger.debug("construct_auth_difference Start!") + # TODO: Make sure we are OK with local_auth or remote_auth having more # auth events in them than strictly necessary. def sort_fun(ev): return ev.depth, ev.event_id + logger.debug("construct_auth_difference after sort_fun!") + # We find the differences by starting at the "bottom" of each list # and iterating up on both lists. The lists are ordered by depth and # then event_id, we iterate up both lists until we find the event ids @@ -823,11 +867,18 @@ class FederationHandler(BaseHandler): local_iter = iter(local_list) remote_iter = iter(remote_list) - current_local = local_iter.next() - current_remote = remote_iter.next() + logger.debug("construct_auth_difference before get_next!") def get_next(it, opt=None): - return it.next() if it.has_next() else opt + try: + return it.next() + except: + return opt + + current_local = get_next(local_iter) + current_remote = get_next(remote_iter) + + logger.debug("construct_auth_difference before while") missing_remotes = [] missing_locals = [] @@ -867,6 +918,8 @@ class FederationHandler(BaseHandler): current_remote = get_next(remote_iter) continue + logger.debug("construct_auth_difference after while") + # missing locals should be sent to the server # We should find why we are missing remotes, as they will have been # rejected. @@ -886,6 +939,7 @@ class FederationHandler(BaseHandler): reason = yield self.store.get_rejection_reason(e.event_id) if reason is None: # FIXME: ERRR?! + logger.warn("Could not find reason for %s", e.event_id) raise RuntimeError("") reason_map[e.event_id] = reason @@ -899,7 +953,10 @@ class FederationHandler(BaseHandler): # TODO: Get proof. pass + logger.debug("construct_auth_difference returning") + defer.returnValue({ + "auth_chain": local_auth, "rejects": { e.event_id: { "reason": reason_map[e.event_id], diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py index b7249700d7..4e1a9a2783 100644 --- a/synapse/storage/rejections.py +++ b/synapse/storage/rejections.py @@ -28,12 +28,12 @@ class RejectionsStore(SQLBaseStore): values={ "event_id": event_id, "reason": reason, - "last_failure": self._clock.time_msec(), + "last_check": self._clock.time_msec(), } ) def get_rejection_reason(self, event_id): - self._simple_select_one_onecol( + return self._simple_select_one_onecol( table="rejections", retcol="reason", keyvalues={ diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index ed21defd13..44dbce6bea 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -52,6 +52,7 @@ class FederationTestCase(unittest.TestCase): "get_room", "get_destination_retry_timings", "set_destination_retry_timings", + "have_events", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -90,6 +91,7 @@ class FederationTestCase(unittest.TestCase): self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) + self.datastore.have_events.return_value = defer.succeed({}) def annotate(ev, old_state=None): context = Mock() -- cgit 1.4.1 From e0d2c6889bf31cf5f48e77334fed23352d19a75d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 17:04:31 +0000 Subject: Allow kind to be set to null to delete a pusher. --- synapse/rest/client/v1/pusher.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 72d5e9e476..353a4a6589 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -31,6 +31,16 @@ class PusherRestServlet(ClientV1RestServlet): content = _parse_json(request) + pusher_pool = self.hs.get_pusherpool() + + if ('pushkey' in content and 'app_id' in content + and 'kind' in content and + content['kind'] is None): + yield pusher_pool.remove_pusher( + content['app_id'], content['pushkey'] + ) + defer.returnValue((200, {})) + reqd = ['instance_handle', 'kind', 'app_id', 'app_display_name', 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] @@ -41,7 +51,6 @@ class PusherRestServlet(ClientV1RestServlet): raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) - pusher_pool = self.hs.get_pusherpool() try: yield pusher_pool.add_pusher( user_name=user.to_string(), -- cgit 1.4.1 From 4bdfce30d70dddaa7c6de551fe3c9eed4a899d49 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 17:12:11 +0000 Subject: Renumber priority classes so we can use 0 for defaults. --- synapse/rest/client/v1/push_rule.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2b1e930326..0f78fa667c 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -26,11 +26,11 @@ import json class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") PRIORITY_CLASS_MAP = { - 'underride': 0, - 'sender': 1, - 'room': 2, - 'content': 3, - 'override': 4, + 'underride': 1, + 'sender': 2, + 'room': 3, + 'content': 4, + 'override': 5, } PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( -- cgit 1.4.1 From 93ed31dda2e23742c3d7f3eee6ac6839682f0ce9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 17:41:48 +0000 Subject: Create a separate filter object to do the actual filtering, so that we can split the storage and management of filters from the actual filter code and don't have to load a filter from the db each time we filter an event --- synapse/api/filtering.py | 220 ++++++++++++++++----------------- synapse/rest/client/v2_alpha/filter.py | 2 +- tests/api/test_filtering.py | 108 ++++++++-------- 3 files changed, 166 insertions(+), 164 deletions(-) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index e16c0e559f..b7e5d3222f 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -25,127 +25,25 @@ class Filtering(object): self.store = hs.get_datastore() def get_user_filter(self, user_localpart, filter_id): - return self.store.get_user_filter(user_localpart, filter_id) + result = self.store.get_user_filter(user_localpart, filter_id) + result.addCallback(Filter) + return result def add_user_filter(self, user_localpart, user_filter): self._check_valid_filter(user_filter) return self.store.add_user_filter(user_localpart, user_filter) - def filter_public_user_data(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["public_user_data"] - ) - - def filter_private_user_data(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["private_user_data"] - ) - - def filter_room_state(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["room", "state"] - ) - - def filter_room_events(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["room", "events"] - ) - - def filter_room_ephemeral(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["room", "ephemeral"] - ) - # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for # them however - @defer.inlineCallbacks - def _filter_on_key(self, events, user, filter_id, keys): - filter_json = yield self.get_user_filter(user.localpart, filter_id) - if not filter_json: - defer.returnValue(events) - - try: - # extract the right definition from the filter - definition = filter_json - for key in keys: - definition = definition[key] - defer.returnValue(self._filter_with_definition(events, definition)) - except KeyError: - # return all events if definition isn't specified. - defer.returnValue(events) - - def _filter_with_definition(self, events, definition): - return [e for e in events if self._passes_definition(definition, e)] - - def _passes_definition(self, definition, event): - """Check if the event passes through the given definition. - - Args: - definition(dict): The definition to check against. - event(Event): The event to check. - Returns: - True if the event passes through the filter. - """ - # Algorithm notes: - # For each key in the definition, check the event meets the criteria: - # * For types: Literal match or prefix match (if ends with wildcard) - # * For senders/rooms: Literal match only - # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' - # and 'not_types' then it is treated as only being in 'not_types') - - # room checks - if hasattr(event, "room_id"): - room_id = event.room_id - allow_rooms = definition.get("rooms", None) - reject_rooms = definition.get("not_rooms", None) - if reject_rooms and room_id in reject_rooms: - return False - if allow_rooms and room_id not in allow_rooms: - return False - - # sender checks - if hasattr(event, "sender"): - # Should we be including event.state_key for some event types? - sender = event.sender - allow_senders = definition.get("senders", None) - reject_senders = definition.get("not_senders", None) - if reject_senders and sender in reject_senders: - return False - if allow_senders and sender not in allow_senders: - return False - - # type checks - if "not_types" in definition: - for def_type in definition["not_types"]: - if self._event_matches_type(event, def_type): - return False - if "types" in definition: - included = False - for def_type in definition["types"]: - if self._event_matches_type(event, def_type): - included = True - break - if not included: - return False - - return True - - def _event_matches_type(self, event, def_type): - if def_type.endswith("*"): - type_prefix = def_type[:-1] - return event.type.startswith(type_prefix) - else: - return event.type == def_type - - def _check_valid_filter(self, user_filter): + def _check_valid_filter(self, user_filter_json): """Check if the provided filter is valid. This inspects all definitions contained within the filter. Args: - user_filter(dict): The filter + user_filter_json(dict): The filter Raises: SynapseError: If the filter is not valid. """ @@ -162,13 +60,13 @@ class Filtering(object): ] for key in top_level_definitions: - if key in user_filter: - self._check_definition(user_filter[key]) + if key in user_filter_json: + self._check_definition(user_filter_json[key]) - if "room" in user_filter: + if "room" in user_filter_json: for key in room_level_definitions: - if key in user_filter["room"]: - self._check_definition(user_filter["room"][key]) + if key in user_filter_json["room"]: + self._check_definition(user_filter_json["room"][key]) def _check_definition(self, definition): """Check if the provided definition is valid. @@ -237,3 +135,101 @@ class Filtering(object): if ("bundle_updates" in definition and type(definition["bundle_updates"]) != bool): raise SynapseError(400, "Bad bundle_updates: expected bool.") + + +class Filter(object): + def __init__(self, filter_json): + self.filter_json = filter_json + + def filter_public_user_data(self, events): + return self._filter_on_key(events, ["public_user_data"]) + + def filter_private_user_data(self, events): + return self._filter_on_key(events, ["private_user_data"]) + + def filter_room_state(self, events): + return self._filter_on_key(events, ["room", "state"]) + + def filter_room_events(self, events): + return self._filter_on_key(events, ["room", "events"]) + + def filter_room_ephemeral(self, events): + return self._filter_on_key(events, ["room", "ephemeral"]) + + def _filter_on_key(self, events, keys): + filter_json = self.filter_json + if not filter_json: + return events + + try: + # extract the right definition from the filter + definition = filter_json + for key in keys: + definition = definition[key] + return self._filter_with_definition(events, definition) + except KeyError: + # return all events if definition isn't specified. + return events + + def _filter_with_definition(self, events, definition): + return [e for e in events if self._passes_definition(definition, e)] + + def _passes_definition(self, definition, event): + """Check if the event passes through the given definition. + + Args: + definition(dict): The definition to check against. + event(Event): The event to check. + Returns: + True if the event passes through the filter. + """ + # Algorithm notes: + # For each key in the definition, check the event meets the criteria: + # * For types: Literal match or prefix match (if ends with wildcard) + # * For senders/rooms: Literal match only + # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' + # and 'not_types' then it is treated as only being in 'not_types') + + # room checks + if hasattr(event, "room_id"): + room_id = event.room_id + allow_rooms = definition.get("rooms", None) + reject_rooms = definition.get("not_rooms", None) + if reject_rooms and room_id in reject_rooms: + return False + if allow_rooms and room_id not in allow_rooms: + return False + + # sender checks + if hasattr(event, "sender"): + # Should we be including event.state_key for some event types? + sender = event.sender + allow_senders = definition.get("senders", None) + reject_senders = definition.get("not_senders", None) + if reject_senders and sender in reject_senders: + return False + if allow_senders and sender not in allow_senders: + return False + + # type checks + if "not_types" in definition: + for def_type in definition["not_types"]: + if self._event_matches_type(event, def_type): + return False + if "types" in definition: + included = False + for def_type in definition["types"]: + if self._event_matches_type(event, def_type): + included = True + break + if not included: + return False + + return True + + def _event_matches_type(self, event, def_type): + if def_type.endswith("*"): + type_prefix = def_type[:-1] + return event.type.startswith(type_prefix) + else: + return event.type == def_type diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index cee06ccaca..6ddc495d23 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -59,7 +59,7 @@ class GetFilterRestServlet(RestServlet): filter_id=filter_id, ) - defer.returnValue((200, filter)) + defer.returnValue((200, filter.filter_json)) except KeyError: raise SynapseError(400, "No such filter") diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index aa93616a9f..babf4c37f1 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -25,6 +25,7 @@ from tests.utils import ( from synapse.server import HomeServer from synapse.types import UserID +from synapse.api.filtering import Filter user_localpart = "test_user" MockEvent = namedtuple("MockEvent", "sender type room_id") @@ -53,6 +54,7 @@ class FilteringTestCase(unittest.TestCase): ) self.filtering = hs.get_filtering() + self.filter = Filter({}) self.datastore = hs.get_datastore() @@ -66,7 +68,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_types_works_with_wildcards(self): @@ -79,7 +81,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_types_works_with_unknowns(self): @@ -92,7 +94,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_works_with_literals(self): @@ -105,7 +107,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_works_with_wildcards(self): @@ -118,7 +120,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_works_with_unknowns(self): @@ -131,7 +133,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_takes_priority_over_types(self): @@ -145,7 +147,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_senders_works_with_literals(self): @@ -158,7 +160,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_senders_works_with_unknowns(self): @@ -171,7 +173,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_senders_works_with_literals(self): @@ -184,7 +186,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_senders_works_with_unknowns(self): @@ -197,7 +199,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_senders_takes_priority_over_senders(self): @@ -211,7 +213,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_rooms_works_with_literals(self): @@ -224,7 +226,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!secretbase:unknown" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_rooms_works_with_unknowns(self): @@ -237,7 +239,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_rooms_works_with_literals(self): @@ -250,7 +252,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_rooms_works_with_unknowns(self): @@ -263,7 +265,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_rooms_takes_priority_over_rooms(self): @@ -277,7 +279,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!secretbase:unknown" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event(self): @@ -295,7 +297,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event_bad_sender(self): @@ -313,7 +315,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event_bad_room(self): @@ -331,7 +333,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!piggyshouse:muppets" # nope ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event_bad_type(self): @@ -349,12 +351,12 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) @defer.inlineCallbacks def test_filter_public_user_data_match(self): - user_filter = { + user_filter_json = { "public_user_data": { "types": ["m.*"] } @@ -362,7 +364,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -371,16 +373,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_public_user_data( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_public_user_data(events=events) self.assertEquals(events, results) @defer.inlineCallbacks def test_filter_public_user_data_no_match(self): - user_filter = { + user_filter_json = { "public_user_data": { "types": ["m.*"] } @@ -388,7 +391,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -397,16 +400,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_public_user_data( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_public_user_data(events=events) self.assertEquals([], results) @defer.inlineCallbacks def test_filter_room_state_match(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -416,7 +420,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -425,16 +429,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_room_state( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_room_state(events=events) self.assertEquals(events, results) @defer.inlineCallbacks def test_filter_room_state_no_match(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -444,7 +449,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -453,16 +458,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_room_state( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_room_state(events) self.assertEquals([], results) @defer.inlineCallbacks def test_add_filter(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -472,11 +478,11 @@ class FilteringTestCase(unittest.TestCase): filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) self.assertEquals(filter_id, 0) - self.assertEquals(user_filter, + self.assertEquals(user_filter_json, (yield self.datastore.get_user_filter( user_localpart=user_localpart, filter_id=0, @@ -485,7 +491,7 @@ class FilteringTestCase(unittest.TestCase): @defer.inlineCallbacks def test_get_filter(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -495,7 +501,7 @@ class FilteringTestCase(unittest.TestCase): filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) filter = yield self.filtering.get_user_filter( @@ -503,4 +509,4 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - self.assertEquals(filter, user_filter) + self.assertEquals(filter.filter_json, user_filter_json) -- cgit 1.4.1 From 365a1867293b8f1d34a1f100d1bc8d2d39bb3d94 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 18:11:28 +0000 Subject: Add basic filtering support --- synapse/rest/client/v2_alpha/sync.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 2ae2eec55d..c1277d2675 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -21,6 +21,7 @@ from synapse.types import StreamToken from synapse.events.utils import ( serialize_event, format_event_for_client_v2_without_event_id, ) +from synapse.api.filtering import Filter from ._base import client_v2_pattern import logging @@ -80,6 +81,7 @@ class SyncRestServlet(RestServlet): self.auth = hs.get_auth() self.sync_handler = hs.get_handlers().sync_handler self.clock = hs.get_clock() + self.filtering = hs.get_filtering() @defer.inlineCallbacks def on_GET(self, request): @@ -109,9 +111,14 @@ class SyncRestServlet(RestServlet): ) # TODO(mjark): Load filter and apply overrides. - # filter = self.filters.load_fitler(filter_id_str) + try: + filter = yield self.filtering.get_user_filter( + user.localpart, filter_id + ) + except: + filter = Filter({}) # filter = filter.apply_overrides(http_request) - # if filter.matches(event): + #if filter.matches(event): # # stuff sync_config = SyncConfig( @@ -121,7 +128,7 @@ class SyncRestServlet(RestServlet): limit=limit, sort=sort, backfill=backfill, - filter="TODO", # TODO(mjark) Add the filter to the config. + filter=filter, ) if since is not None: @@ -162,9 +169,11 @@ class SyncRestServlet(RestServlet): @staticmethod def encode_room(room, filter, time_now, token_id): event_map = {} + state_events = filter.filter_room_state(room.state) + recent_events = filter.filter_room_events(room.events) state_event_ids = [] recent_event_ids = [] - for event in room.state: + for event in state_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( event, time_now, token_id=token_id, @@ -172,7 +181,7 @@ class SyncRestServlet(RestServlet): ) state_event_ids.append(event.event_id) - for event in room.events: + for event in recent_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( event, time_now, token_id=token_id, -- cgit 1.4.1 From ece828a7b72a25502d8daa15f6429c73a620b53c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 18:15:24 +0000 Subject: Update todo for the filtering on sync --- synapse/handlers/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 8c7681a48d..5768702192 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -290,7 +290,8 @@ class SyncHandler(BaseHandler): """ # TODO(mjark): Check if they have joined the room between # the previous sync and this one. - # TODO(mjark): Apply the event filter in sync_config + # TODO(mjark): Apply the event filter in sync_config taking care to get + # enough events to reach the limit # TODO(mjark): Check for redactions we might have missed. recents, token = yield self.store.get_recent_events_for_room( room_id, -- cgit 1.4.1 From 1235f7f383617a91b4b96e287cc1ad205b80a5de Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 18:38:22 +0000 Subject: Add default push rules including setting a sound for messages mentioning your username / display name --- synapse/push/__init__.py | 47 +++++++++++++++++++++++++++++++++++++++-------- synapse/push/baserules.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 synapse/push/baserules.py (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 472ede5480..d19e13d644 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -16,9 +16,10 @@ from twisted.internet import defer from synapse.streams.config import PaginationConfig -from synapse.types import StreamToken +from synapse.types import StreamToken, UserID import synapse.util.async +import baserules import logging import fnmatch @@ -75,14 +76,34 @@ class Pusher(object): rules = yield self.store.get_push_rules_for_user_name(self.user_name) + for r in rules: + r['conditions'] = json.loads(r['conditions']) + r['actions'] = json.loads(r['actions']) + + user_name_localpart = UserID.from_string(self.user_name).localpart + + rules.extend(baserules.make_base_rules(user_name_localpart)) + + # get *our* member event for display name matching + member_events_for_room = yield self.store.get_current_state( + room_id=ev['room_id'], + event_type='m.room.member', + state_key=self.user_name + ) + my_display_name = None + if len(member_events_for_room) > 0: + my_display_name = member_events_for_room[0].content['displayname'] + for r in rules: matches = True - conditions = json.loads(r['conditions']) - actions = json.loads(r['actions']) + conditions = r['conditions'] + actions = r['actions'] for c in conditions: - matches &= self._event_fulfills_condition(ev, c) + matches &= self._event_fulfills_condition( + ev, c, display_name=my_display_name + ) # ignore rules with no actions (we have an explict 'dont_notify' if len(actions) == 0: logger.warn( @@ -95,7 +116,7 @@ class Pusher(object): defer.returnValue(Pusher.DEFAULT_ACTIONS) - def _event_fulfills_condition(self, ev, condition): + def _event_fulfills_condition(self, ev, condition, display_name): if condition['kind'] == 'event_match': if 'pattern' not in condition: logger.warn("event_match condition with no pattern") @@ -103,13 +124,23 @@ class Pusher(object): pat = condition['pattern'] val = _value_for_dotted_key(condition['key'], ev) - if fnmatch.fnmatch(val, pat): - return True - return False + if val is None: + return False + return fnmatch.fnmatch(val.upper(), pat.upper()) elif condition['kind'] == 'device': if 'instance_handle' not in condition: return True return condition['instance_handle'] == self.instance_handle + elif condition['kind'] == 'contains_display_name': + # This is special because display names can be different + # between rooms and so you can't really hard code it in a rule. + # Optimisation: we should cache these names and update them from + # the event stream. + if 'content' not in ev or 'body' not in ev['content']: + return False + return fnmatch.fnmatch( + ev['content']['body'].upper(), "*%s*" % (display_name.upper(),) + ) else: return True diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py new file mode 100644 index 0000000000..4caf7beed2 --- /dev/null +++ b/synapse/push/baserules.py @@ -0,0 +1,35 @@ +def make_base_rules(user_name): + """ + Nominally we reserve priority class 0 for these rules, although + in practice we just append them to the end so we don't actually need it. + """ + return [ + { + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'content.body', + 'pattern': '*%s*' % (user_name,), # Matrix ID match + } + ], + 'actions': [ + 'notify', + { + 'set_sound': 'default' + } + ] + }, + { + 'conditions': [ + { + 'kind': 'contains_display_name' + } + ], + 'actions': [ + 'notify', + { + 'set_sound': 'default' + } + ] + }, + ] \ No newline at end of file -- cgit 1.4.1 From 0b1688639750e2401571263c127817d0b0a43644 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 18:51:22 +0000 Subject: Change 'from' in notification pokes to 'sender' to match client API v2. Send sender display names where they exist. --- synapse/push/__init__.py | 10 ++++++++++ synapse/push/httppusher.py | 9 ++++----- 2 files changed, 14 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index d19e13d644..19478c72a2 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -154,6 +154,16 @@ class Pusher(object): if name_aliases[0] is not None: ctx['name'] = name_aliases[0] + their_member_events_for_room = yield self.store.get_current_state( + room_id=ev['room_id'], + event_type='m.room.member', + state_key=ev['user_id'] + ) + if len(their_member_events_for_room) > 0: + dn = their_member_events_for_room[0].content['displayname'] + if dn is not None: + ctx['sender_display_name'] = dn + defer.returnValue(ctx) @defer.inlineCallbacks diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index ab128e31e5..ac7c3148d7 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -67,10 +67,7 @@ class HttpPusher(Pusher): 'notification': { 'id': event['event_id'], 'type': event['type'], - 'from': event['user_id'], - # we may have to fetch this over federation and we - # can't trust it anyway: is it worth it? - #'from_display_name': 'Steve Stevington' + 'sender': event['user_id'], 'counts': { # -- we don't mark messages as read yet so # we have no way of knowing # Just set the badge to 1 until we have read receipts @@ -93,6 +90,8 @@ class HttpPusher(Pusher): if len(ctx['aliases']): d['notification']['room_alias'] = ctx['aliases'][0] + if 'sender_display_name' in ctx: + d['notification']['sender_display_name'] = ctx['sender_display_name'] if 'name' in ctx: d['notification']['room_name'] = ctx['name'] @@ -119,7 +118,7 @@ class HttpPusher(Pusher): 'notification': { 'id': '', 'type': None, - 'from': '', + 'sender': '', 'counts': { 'unread': 0, 'missed_calls': 0 -- cgit 1.4.1 From fc946f3b8da8c7f71a9c25bf542c04472147bc5b Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 21:59:17 +0000 Subject: Include content in notification pokes --- synapse/push/httppusher.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse') diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index ac7c3148d7..d4c5f03b01 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -87,6 +87,8 @@ class HttpPusher(Pusher): } if event['type'] == 'm.room.member': d['notification']['membership'] = event['content']['membership'] + if 'content' in event: + d['notification']['content'] = event['content'] if len(ctx['aliases']): d['notification']['room_alias'] = ctx['aliases'][0] -- cgit 1.4.1 From c1c7b398270cc19fcf9410b9159f741092e0510b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 10:30:54 +0000 Subject: Fix bug where we changes in outlier in metadata dict propogated to other events --- synapse/events/__init__.py | 2 +- synapse/events/builder.py | 5 +++-- synapse/events/utils.py | 5 ++++- 3 files changed, 8 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 4252e5ab5c..bf07951027 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -18,7 +18,7 @@ from synapse.util.frozenutils import freeze, unfreeze class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): - self.__dict__ = internal_metadata_dict + self.__dict__ = dict(internal_metadata_dict) def get_dict(self): return dict(self.__dict__) diff --git a/synapse/events/builder.py b/synapse/events/builder.py index a9b1b99a10..9d45bdb892 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -23,14 +23,15 @@ import copy class EventBuilder(EventBase): - def __init__(self, key_values={}): + def __init__(self, key_values={}, internal_metadata_dict={}): signatures = copy.deepcopy(key_values.pop("signatures", {})) unsigned = copy.deepcopy(key_values.pop("unsigned", {})) super(EventBuilder, self).__init__( key_values, signatures=signatures, - unsigned=unsigned + unsigned=unsigned, + internal_metadata_dict=internal_metadata_dict, ) def build(self): diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 10a6b9f264..08d6d6fa47 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -88,7 +88,10 @@ def prune_event(event): if "age_ts" in event.unsigned: allowed_fields["unsigned"]["age_ts"] = event.unsigned["age_ts"] - return type(event)(allowed_fields) + return type(event)( + allowed_fields, + internal_metadata_dict=event.internal_metadata.get_dict() + ) def serialize_event(hs, e, client_event=True): -- cgit 1.4.1 From c1d860870b418b9583078022d0babe557b73760f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 10:48:47 +0000 Subject: Fix regression where we no longer correctly handled the case of gaps in our event graph --- synapse/federation/federation_server.py | 3 +++ synapse/handlers/federation.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 8cff4e6472..845a07a3a3 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -366,10 +366,13 @@ class FederationServer(object): logger.debug("Processed pdu %s", event_id) else: logger.warn("Failed to get PDU %s", event_id) + fetch_state = True except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") fetch_state = True + else: + fetch_state = True else: fetch_state = True diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 14c26d8cea..de47a97e6b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -119,7 +119,7 @@ class FederationHandler(BaseHandler): event.room_id, self.server_name ) - if not is_in_room and not event.internal_metadata.outlier: + if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") replication = self.replication_layer @@ -780,6 +780,7 @@ class FederationHandler(BaseHandler): (e.type, e.state_key): e for e in remote_auth_chain if e.event_id in auth_ids } + e.internal_metadata.outlier = True yield self._handle_new_event( origin, e, auth_events=auth ) @@ -787,6 +788,8 @@ class FederationHandler(BaseHandler): except AuthError: pass + # FIXME: Assumes we have and stored all the state for all the + # prev_events current_state = set(e.event_id for e in auth_events.values()) different_auth = event_auth_events - current_state @@ -814,6 +817,7 @@ class FederationHandler(BaseHandler): (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } + e.internal_metadata.outlier = True yield self._handle_new_event( origin, e, auth_events=auth ) @@ -882,7 +886,7 @@ class FederationHandler(BaseHandler): missing_remotes = [] missing_locals = [] - while current_local and current_remote: + while current_local or current_remote: if current_remote is None: missing_locals.append(current_local) current_local = get_next(local_iter) -- cgit 1.4.1 From 823999716eb506740b34d87fcdaaf3ebbe4f760c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:07:57 +0000 Subject: Fix bug in timeout handling in keyclient --- synapse/crypto/keyclient.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 9c910fa3fc..cdb6279764 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -61,9 +61,11 @@ class SynapseKeyClientProtocol(HTTPClient): def __init__(self): self.remote_key = defer.Deferred() + self.host = None def connectionMade(self): - logger.debug("Connected to %s", self.transport.getHost()) + self.host = self.transport.getHost() + logger.debug("Connected to %s", self.host) self.sendCommand(b"GET", b"/_matrix/key/v1/") self.endHeaders() self.timer = reactor.callLater( @@ -92,8 +94,7 @@ class SynapseKeyClientProtocol(HTTPClient): self.timer.cancel() def on_timeout(self): - logger.debug("Timeout waiting for response from %s", - self.transport.getHost()) + logger.debug("Timeout waiting for response from %s", self.host) self.remote_key.errback(IOError("Timeout waiting for response")) self.transport.abortConnection() -- cgit 1.4.1 From 0c2d245fdf5f24f99138f1c60c4e6bed52cf5d5a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:08:52 +0000 Subject: Update the current state of an event if we update auth events. --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index de47a97e6b..cc22f21cd1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -706,7 +706,6 @@ class FederationHandler(BaseHandler): event.event_id, e.msg ) - # TODO: Store rejection. context.rejected = RejectedReason.AUTH_ERROR yield self.store.persist_event( @@ -828,6 +827,9 @@ class FederationHandler(BaseHandler): # 4. Look at rejects and their proofs. # TODO. + context.current_state.update(auth_events) + context.state_group = None + try: self.auth.check(event, auth_events=auth_events) except AuthError: -- cgit 1.4.1 From bd03947c05f23e4adb6ce1a745d725a106fed66c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 11:13:42 +0000 Subject: We do need Twisted 14, not 15: we use internal Twisted things that have been removed in 15. --- setup.py | 2 +- synapse/python_dependencies.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/setup.py b/setup.py index 043cd044a7..3249e87a96 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( install_requires=[ "syutil==0.0.2", "matrix_angular_sdk==0.6.0", - "Twisted>=14.0.0", + "Twisted==14.0.2", "service_identity>=1.0.0", "pyopenssl>=0.14", "pyyaml", diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 4182ad990f..826a36f203 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted>=14.0.0": ["twisted>=14.0.0"], + "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], -- cgit 1.4.1 From 2c9e136d5721d1797e153268f4562c27887a6201 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:14:33 +0000 Subject: Fix bad merge fo python_dependencies.py --- synapse/python_dependencies.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ba9308803c..4182ad990f 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,6 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "matrix_angular_sdk>=0.6.0": ["syweb>=0.6.0"], "Twisted>=14.0.0": ["twisted>=14.0.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], -- cgit 1.4.1 From 22dd1cde2d83a2448074816108b85d1957315236 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:32:35 +0000 Subject: Filter the recent events before applying the limit when doing an incremental sync with a gap --- synapse/api/filtering.py | 2 -- synapse/handlers/sync.py | 53 ++++++++++++++++++++++++++---------- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/storage/stream.py | 21 ++++++++++---- 4 files changed, 54 insertions(+), 24 deletions(-) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index b7e5d3222f..fa4de2614d 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5768702192..0df1851b0e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -278,6 +278,40 @@ class SyncHandler(BaseHandler): next_batch=now_token, )) + @defer.inlineCallbacks + def load_filtered_recents(self, room_id, sync_config, since_token, + now_token): + limited = True + recents = [] + filtering_factor = 2 + load_limit = max(sync_config.limit * filtering_factor, 100) + max_repeat = 3 # Only try a few times per room, otherwise + room_key = now_token.room_key + + while limited and len(recents) < sync_config.limit and max_repeat: + events, room_key = yield self.store.get_recent_events_for_room( + room_id, + limit=load_limit + 1, + from_token=since_token.room_key, + end_token=room_key, + ) + loaded_recents = sync_config.filter.filter_room_events(events) + loaded_recents.extend(recents) + recents = loaded_recents + if len(events) <= load_limit: + limited = False + max_repeat -= 1 + + if len(recents) > sync_config.limit: + recents = recents[-sync_config.limit:] + room_key = recents[0].internal_metadata.before + + prev_batch_token = now_token.copy_and_replace( + "room_key", room_key + ) + + defer.returnValue((recents, prev_batch_token, limited)) + @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, @@ -288,28 +322,17 @@ class SyncHandler(BaseHandler): Returns: A Deferred RoomSyncResult """ + # TODO(mjark): Check if they have joined the room between # the previous sync and this one. - # TODO(mjark): Apply the event filter in sync_config taking care to get - # enough events to reach the limit # TODO(mjark): Check for redactions we might have missed. - recents, token = yield self.store.get_recent_events_for_room( - room_id, - limit=sync_config.limit + 1, - from_token=since_token.room_key, - end_token=now_token.room_key, + + recents, prev_batch_token, limited = self.load_filtered_recents( + room_id, sync_config, since_token, ) logging.debug("Recents %r", recents) - if len(recents) > sync_config.limit: - limited = True - recents = recents[1:] - else: - limited = False - - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - # TODO(mjark): This seems racy since this isn't being passed a # token to indicate what point in the stream this is current_state_events = yield self.state_handler.get_current_state( diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index c1277d2675..46ea50d118 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -116,7 +116,7 @@ class SyncRestServlet(RestServlet): user.localpart, filter_id ) except: - filter = Filter({}) + filter = Filter({}) # filter = filter.apply_overrides(http_request) #if filter.matches(event): # # stuff diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 2ea5e1a021..73504c8b52 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -181,15 +181,11 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) - for event, row in zip(ret, rows): - stream = row["stream_ordering"] - topo = event.depth - internal = event.internal_metadata - internal.before = str(_StreamToken(topo, stream - 1)) - internal.after = str(_StreamToken(topo, stream)) + self._set_before_and_after(ret, rows) if rows: key = "s%d" % max([r["stream_ordering"] for r in rows]) + else: # Assume we didn't get anything because there was nothing to # get. @@ -267,6 +263,8 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + self._set_before_and_after(events, rows) + return events, next_token, return self.runInteraction("paginate_room_events", f) @@ -328,6 +326,8 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + self._set_before_and_after(events, rows) + return events, token return self.runInteraction( @@ -354,3 +354,12 @@ class StreamStore(SQLBaseStore): key = res[0]["m"] return "s%d" % (key,) + + @staticmethod + def _set_before_and_after(events, rows): + for event, row in zip(events, rows): + stream = row["stream_ordering"] + topo = event.depth + internal = event.internal_metadata + internal.before = str(_StreamToken(topo, stream - 1)) + internal.after = str(_StreamToken(topo, stream)) -- cgit 1.4.1 From e97de6d96a25912b34fa38cd3a587d798f6b676c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:35:20 +0000 Subject: Filter the recent events before applying the limit when doing an initial sync --- synapse/handlers/sync.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 0df1851b0e..b83fcad655 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -163,12 +163,11 @@ class SyncHandler(BaseHandler): Returns: A Deferred RoomSyncResult. """ - recent_events, token = yield self.store.get_recent_events_for_room( - room_id, - limit=sync_config.limit, - end_token=now_token.room_key, + + recents, prev_batch_token, limited = self.load_filtered_recents( + room_id, sync_config, now_token, ) - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state_events = yield self.state_handler.get_current_state( room_id ) @@ -176,10 +175,10 @@ class SyncHandler(BaseHandler): defer.returnValue(RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, - events=recent_events, + events=recents, prev_batch=prev_batch_token, state=current_state_events, - limited=True, + limited=limited, ephemeral=[], )) @@ -279,8 +278,8 @@ class SyncHandler(BaseHandler): )) @defer.inlineCallbacks - def load_filtered_recents(self, room_id, sync_config, since_token, - now_token): + def load_filtered_recents(self, room_id, sync_config, now_token, + since_token=None): limited = True recents = [] filtering_factor = 2 @@ -292,7 +291,7 @@ class SyncHandler(BaseHandler): events, room_key = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, - from_token=since_token.room_key, + from_token=since_token.room_key if since_token else None, end_token=room_key, ) loaded_recents = sync_config.filter.filter_room_events(events) @@ -328,7 +327,7 @@ class SyncHandler(BaseHandler): # TODO(mjark): Check for redactions we might have missed. recents, prev_batch_token, limited = self.load_filtered_recents( - room_id, sync_config, since_token, + room_id, sync_config, now_token, since_token, ) logging.debug("Recents %r", recents) -- cgit 1.4.1 From 8498d348d818aa2d2cb9bb9bb2775103840f355d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:42:09 +0000 Subject: Fix token formatting --- synapse/handlers/sync.py | 6 +++--- synapse/storage/stream.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b83fcad655..3c68e2a9ec 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -164,7 +164,7 @@ class SyncHandler(BaseHandler): A Deferred RoomSyncResult. """ - recents, prev_batch_token, limited = self.load_filtered_recents( + recents, prev_batch_token, limited = yield self.load_filtered_recents( room_id, sync_config, now_token, ) @@ -288,7 +288,7 @@ class SyncHandler(BaseHandler): room_key = now_token.room_key while limited and len(recents) < sync_config.limit and max_repeat: - events, room_key = yield self.store.get_recent_events_for_room( + events, (room_key,_) = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, @@ -326,7 +326,7 @@ class SyncHandler(BaseHandler): # the previous sync and this one. # TODO(mjark): Check for redactions we might have missed. - recents, prev_batch_token, limited = self.load_filtered_recents( + recents, prev_batch_token, limited = yield self.load_filtered_recents( room_id, sync_config, now_token, since_token, ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 73504c8b52..3ccb6f8a61 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -316,9 +316,9 @@ class StreamStore(SQLBaseStore): toke = rows[0]["stream_ordering"] - 1 start_token = str(_StreamToken(topo, toke)) - token = (start_token, end_token) + token = (start_token, str(end_token)) else: - token = (end_token, end_token) + token = (str(end_token), str(end_token)) events = self._get_events_txn( txn, -- cgit 1.4.1 From c562f237f6236c981f2e7858ff2748f62bd63ad1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:43:00 +0000 Subject: Unused import --- synapse/api/filtering.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index b7e5d3222f..fa4de2614d 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID -- cgit 1.4.1 From 4a67834bc84f604605c618049599f4638434c7cf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:50:15 +0000 Subject: Pass client info to the sync_config --- synapse/handlers/sync.py | 5 +++-- synapse/rest/client/v2_alpha/sync.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3c68e2a9ec..1a74a4c97c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", - "device", + "client_info", "limit", "gap", "sort", @@ -288,12 +288,13 @@ class SyncHandler(BaseHandler): room_key = now_token.room_key while limited and len(recents) < sync_config.limit and max_repeat: - events, (room_key,_) = yield self.store.get_recent_events_for_room( + events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, end_token=room_key, ) + (room_key, _) = keys loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 46ea50d118..81d5cf8ead 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -123,7 +123,7 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, - device="TODO", # TODO(mjark) Get the device_id from access_token + client_info=client, gap=gap, limit=limit, sort=sort, -- cgit 1.4.1 From a70a801184814d116ed5b10a952e17c45df7bfc8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 13:34:01 +0000 Subject: Fix bug where we superfluously asked for current state. Change API of /query_auth/ so that we don't duplicate events in the response. --- synapse/api/auth.py | 2 ++ synapse/federation/federation_client.py | 7 +---- synapse/federation/federation_server.py | 12 ++++---- synapse/handlers/federation.py | 51 ++++++++++++--------------------- synapse/state.py | 20 ++++++++++--- 5 files changed, 43 insertions(+), 49 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3471afd7e7..37e31d2b6f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -102,6 +102,8 @@ class Auth(object): def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) + logger.debug("Got curr_state %s", curr_state) + for event in curr_state: if event.type == EventTypes.Member: try: diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 1173ca817b..e1539bd0e0 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -357,15 +357,10 @@ class FederationClient(object): for e in content["auth_chain"] ] - missing = [ - (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) - for e in content.get("missing", []) - ] - ret = { "auth_chain": auth_chain, "rejects": content.get("rejects", []), - "missing": missing, + "missing": content.get("missing", []), } defer.returnValue(ret) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 845a07a3a3..84ed0a0ba0 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -252,11 +252,8 @@ class FederationServer(object): e.get_pdu_json(time_now) for e in ret["auth_chain"] ], - "rejects": content.get("rejects", []), - "missing": [ - e.get_pdu_json(time_now) - for e in ret.get("missing", []) - ], + "rejects": ret.get("rejects", []), + "missing": ret.get("missing", []), } defer.returnValue( @@ -372,7 +369,10 @@ class FederationServer(object): logger.exception("Failed to get PDU") fetch_state = True else: - fetch_state = True + prevs = {e_id for e_id, _ in pdu.prev_events} + seen = set(have_seen.keys()) + if prevs - seen: + fetch_state = True else: fetch_state = True diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index cc22f21cd1..35cad4182a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -121,38 +121,18 @@ class FederationHandler(BaseHandler): ) if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") - - replication = self.replication_layer - - if not state: - state, auth_chain = yield replication.get_state_for_room( - origin, room_id=event.room_id, event_id=event.event_id, - ) - - if not auth_chain: - auth_chain = yield replication.get_event_auth( - origin, - context=event.room_id, - event_id=event.event_id, - ) - - for e in auth_chain: - e.internal_metadata.outlier = True - try: - yield self._handle_new_event(origin, e) - except: - logger.exception( - "Failed to handle auth event %s", - e.event_id, - ) - current_state = state - if state: + if state and auth_chain is not None: for e in state: e.internal_metadata.outlier = True try: - yield self._handle_new_event(origin, e) + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event(origin, e, auth_events=auth) except: logger.exception( "Failed to handle state event %s", @@ -809,18 +789,23 @@ class FederationHandler(BaseHandler): ) # 3. Process any remote auth chain events we haven't seen. - for e in result.get("missing", []): + for missing_id in result.get("missing", []): try: - auth_ids = [e_id for e_id, _ in e.auth_events] + for e in result["auth_chain"]: + if e.event_id == missing_id: + ev = e + break + + auth_ids = [e_id for e_id, _ in ev.auth_events] auth = { (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } - e.internal_metadata.outlier = True + ev.internal_metadata.outlier = True yield self._handle_new_event( - origin, e, auth_events=auth + origin, ev, auth_events=auth ) - auth_events[(e.type, e.state_key)] = e + auth_events[(ev.type, ev.state_key)] = ev except AuthError: pass @@ -970,5 +955,5 @@ class FederationHandler(BaseHandler): } for e in base_remote_rejected }, - "missing": missing_locals, + "missing": [e.event_id for e in missing_locals], }) diff --git a/synapse/state.py b/synapse/state.py index d9fdfb34be..e6632978b5 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -166,10 +166,17 @@ class StateHandler(object): first is the name of a state group if one and only one is involved, otherwise `None`. """ + logger.debug("resolve_state_groups event_ids %s", event_ids) + state_groups = yield self.store.get_state_groups( event_ids ) + logger.debug( + "resolve_state_groups state_groups %s", + state_groups.keys() + ) + group_names = set(state_groups.keys()) if len(group_names) == 1: name, state_list = state_groups.items().pop() @@ -205,6 +212,15 @@ class StateHandler(object): if len(v.values()) > 1 } + logger.debug( + "resolve_state_groups Unconflicted state: %s", + unconflicted_state.values(), + ) + logger.debug( + "resolve_state_groups Conflicted state: %s", + conflicted_state.values(), + ) + if event_type: prev_states_events = conflicted_state.get( (event_type, state_key), [] @@ -240,10 +256,6 @@ class StateHandler(object): 1. power levels 2. memberships 3. other events. - - :param conflicted_state: - :param auth_events: - :return: """ resolved_state = {} power_key = (EventTypes.PowerLevels, "") -- cgit 1.4.1 From 8fe39a03119b3d092e84319d6d99b6ffc49a83f5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 13:33:41 +0000 Subject: Check if the user has joined the room between incremental syncs --- synapse/handlers/sync.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1a74a4c97c..9e07f30b24 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -16,7 +16,7 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig -from synapse.api.constants import Membership +from synapse.api.constants import Membership, EventTypes from twisted.internet import defer @@ -250,6 +250,11 @@ class SyncHandler(BaseHandler): ) else: prev_batch = now_token + + state = yield self.check_joined_room( + sync_config, room_id, state + ) + room_sync = RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, @@ -323,8 +328,6 @@ class SyncHandler(BaseHandler): A Deferred RoomSyncResult """ - # TODO(mjark): Check if they have joined the room between - # the previous sync and this one. # TODO(mjark): Check for redactions we might have missed. recents, prev_batch_token, limited = yield self.load_filtered_recents( @@ -349,6 +352,10 @@ class SyncHandler(BaseHandler): current_state=current_state_events, ) + state_events_delta = yield self.check_joined_room( + sync_config, room_id, state_events_delta + ) + room_sync = RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, @@ -356,7 +363,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, - typing=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, None) ) logging.debug("Room sync: %r", room_sync) @@ -402,3 +409,19 @@ class SyncHandler(BaseHandler): if event.event_id not in previous_dict: state_delta.append(event) return state_delta + + @defer.inlineCallbacks + def check_joined_room(self, sync_config, room_id, state_delta): + joined = False + for event in state_delta: + if ( + event.type == EventTypes.Member + and event.state_key == sync_config.user.to_string() + ): + if event.content["membership"] == Membership.JOIN: + joined = True + + if joined: + state_delta = yield self.state_handler.get_current_state(room_id) + + defer.returnValue(state_delta) -- cgit 1.4.1 From 76d7fd39cd44393a5c712930e77c64e202df17cc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 13:52:02 +0000 Subject: Style changes. --- synapse/state.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index d9fdfb34be..43bda35253 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -37,6 +37,9 @@ def _get_state_key_from_event(event): KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) +AuthEventTypes = (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) + + class StateHandler(object): """ Responsible for doing state conflict resolution. """ @@ -215,7 +218,7 @@ class StateHandler(object): auth_events = { k: e for k, e in unconflicted_state.items() - if k[0] in (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) + if k[0] in AuthEventTypes } try: @@ -240,10 +243,6 @@ class StateHandler(object): 1. power levels 2. memberships 3. other events. - - :param conflicted_state: - :param auth_events: - :return: """ resolved_state = {} power_key = (EventTypes.PowerLevels, "") @@ -305,4 +304,4 @@ class StateHandler(object): def key_func(e): return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() - return sorted(events, key=key_func) \ No newline at end of file + return sorted(events, key=key_func) -- cgit 1.4.1 From 7a9f6f083e0998893eb9d837512009509d81c998 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 13:55:46 +0000 Subject: Remove commented line --- synapse/state.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 43bda35253..dff11711a6 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -46,7 +46,6 @@ class StateHandler(object): def __init__(self, hs): self.store = hs.get_datastore() - # self.auth = hs.get_auth() self.hs = hs @defer.inlineCallbacks -- cgit 1.4.1 From 3d7026e709b145378a6b8a37404910951ef682b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:37:19 +0000 Subject: Add a slightly more helpful comment --- synapse/state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index dff11711a6..081bc31bb5 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -296,7 +296,8 @@ class StateHandler(object): except AuthError: pass - # Oh dear. + # Use the last event (the one with the least depth) if they all fail + # the auth check. return event def _ordered_events(self, events): -- cgit 1.4.1 From 322a047502c938bfe9a6acab47e370e69fefc522 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 14:46:03 +0000 Subject: Add room member count condition and default rule to make a noise on rooms of only 2 people. --- synapse/push/__init__.py | 50 ++++++++++++++++++++++++++++++++++++++++----- synapse/push/baserules.py | 14 +++++++++++++ synapse/storage/__init__.py | 5 ++++- 3 files changed, 63 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 19478c72a2..cc05278c8c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -24,6 +24,7 @@ import baserules import logging import fnmatch import json +import re logger = logging.getLogger(__name__) @@ -34,6 +35,8 @@ class Pusher(object): GIVE_UP_AFTER = 24 * 60 * 60 * 1000 DEFAULT_ACTIONS = ['notify'] + INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$") + def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): @@ -88,11 +91,21 @@ class Pusher(object): member_events_for_room = yield self.store.get_current_state( room_id=ev['room_id'], event_type='m.room.member', - state_key=self.user_name + state_key=None ) my_display_name = None - if len(member_events_for_room) > 0: - my_display_name = member_events_for_room[0].content['displayname'] + room_member_count = 0 + for mev in member_events_for_room: + if mev.content['membership'] != 'join': + continue + + # This loop does two things: + # 1) Find our current display name + if mev.state_key == self.user_name: + my_display_name = mev.content['displayname'] + + # and 2) Get the number of people in that room + room_member_count += 1 for r in rules: matches = True @@ -102,7 +115,8 @@ class Pusher(object): for c in conditions: matches &= self._event_fulfills_condition( - ev, c, display_name=my_display_name + ev, c, display_name=my_display_name, + room_member_count=room_member_count ) # ignore rules with no actions (we have an explict 'dont_notify' if len(actions) == 0: @@ -116,7 +130,7 @@ class Pusher(object): defer.returnValue(Pusher.DEFAULT_ACTIONS) - def _event_fulfills_condition(self, ev, condition, display_name): + def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): if condition['kind'] == 'event_match': if 'pattern' not in condition: logger.warn("event_match condition with no pattern") @@ -138,9 +152,35 @@ class Pusher(object): # the event stream. if 'content' not in ev or 'body' not in ev['content']: return False + if not display_name: + return False return fnmatch.fnmatch( ev['content']['body'].upper(), "*%s*" % (display_name.upper(),) ) + elif condition['kind'] == 'room_member_count': + if 'is' not in condition: + return False + m = Pusher.INEQUALITY_EXPR.match(condition['is']) + if not m: + return False + ineq = m.group(1) + rhs = m.group(2) + if not rhs.isdigit(): + return False + rhs = int(rhs) + + if ineq == '' or ineq == '==': + return room_member_count == rhs + elif ineq == '<': + return room_member_count < rhs + elif ineq == '>': + return room_member_count > rhs + elif ineq == '>=': + return room_member_count >= rhs + elif ineq == '<=': + return room_member_count <= rhs + else: + return False else: return True diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 4caf7beed2..bd162baade 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -32,4 +32,18 @@ def make_base_rules(user_name): } ] }, + { + 'conditions': [ + { + 'kind': 'room_member_count', + 'is': '2' + } + ], + 'actions': [ + 'notify', + { + 'set_sound': 'default' + } + ] + } ] \ No newline at end of file diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 277581b4e2..7b18acf421 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -375,9 +375,12 @@ class DataStore(RoomMemberStore, RoomStore, "redacted": del_sql, } - if event_type: + if event_type and state_key is not None: sql += " AND s.type = ? AND s.state_key = ? " args = (room_id, event_type, state_key) + elif event_type: + sql += " AND s.type = ?" + args = (room_id, event_type) else: args = (room_id, ) -- cgit 1.4.1 From 472cf532b7e64a30c5d7ecdeec9b8f89abf8276c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:48:03 +0000 Subject: Put CREATE rejections into seperate .sql --- synapse/storage/schema/im.sql | 7 ------- synapse/storage/schema/rejections.sql | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 synapse/storage/schema/rejections.sql (limited to 'synapse') diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index bc7c6b6ed5..dd00c1cd2f 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -123,10 +123,3 @@ CREATE TABLE IF NOT EXISTS room_hosts( ); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); - -CREATE TABLE IF NOT EXISTS rejections( - event_id TEXT NOT NULL, - reason TEXT NOT NULL, - last_check TEXT NOT NULL, - CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE -); diff --git a/synapse/storage/schema/rejections.sql b/synapse/storage/schema/rejections.sql new file mode 100644 index 0000000000..bd2a8b1bb5 --- /dev/null +++ b/synapse/storage/schema/rejections.sql @@ -0,0 +1,21 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE +); -- cgit 1.4.1 From 2f4cb04f455d24d0086b37bc363137e995d908d5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:48:11 +0000 Subject: Be more specific in naming columns in selects. --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1f5e74a16a..b350fd61f1 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -504,7 +504,7 @@ class SQLBaseStore(object): def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): sql = ( - "SELECT internal_metadata, json, r.event_id, reason " + "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " "FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " "LEFT JOIN rejections as rej on rej.event_id = e.event_id " -- cgit 1.4.1 From e97f756a05519f9d5a8a6ff78182b691dd1355df Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 14:54:06 +0000 Subject: Use 'in' to test if the key exists, remove unused _filters_for_user --- synapse/api/filtering.py | 8 ++------ synapse/storage/filtering.py | 4 ---- 2 files changed, 2 insertions(+), 10 deletions(-) (limited to 'synapse') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index fa4de2614d..4d570b74f8 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -114,21 +114,17 @@ class Filtering(object): if not isinstance(event_type, basestring): raise SynapseError(400, "Event type should be a string") - try: + if "format" in definition: event_format = definition["format"] if event_format not in ["federation", "events"]: raise SynapseError(400, "Invalid format: %s" % (event_format,)) - except KeyError: - pass # format is optional - try: + if "select" in definition: event_select_list = definition["select"] for select_key in event_select_list: if select_key not in ["event_id", "origin_server_ts", "thread_id", "content", "content.body"]: raise SynapseError(400, "Bad select: %s" % (select_key,)) - except KeyError: - pass # select is optional if ("bundle_updates" in definition and type(definition["bundle_updates"]) != bool): diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index cb01c2040f..e86eeced45 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -20,10 +20,6 @@ from ._base import SQLBaseStore import json -# TODO(paul) -_filters_for_user = {} - - class FilteringStore(SQLBaseStore): @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): -- cgit 1.4.1 From 4f7fe63b6df891c196698b9d896ca7893c7a8a8e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:57:53 +0000 Subject: Remember to add schema file to list --- synapse/storage/__init__.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f233ff2a2a..d03ee80303 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -70,6 +70,7 @@ SCHEMAS = [ "pusher", "media_repository", "filtering", + "rejections", ] -- cgit 1.4.1 From 91015ad008b0d4538022fbddae7da397f7bd7000 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:58:54 +0000 Subject: Remove merge conflict --- synapse/storage/__init__.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d03ee80303..f35ece6446 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -32,11 +32,8 @@ from .event_federation import EventFederationStore from .pusher import PusherStore from .push_rule import PushRuleStore from .media_repository import MediaRepositoryStore -<<<<<<< HEAD from .rejections import RejectionsStore -======= ->>>>>>> 471c47441d0c188e845b75c8f446c44899fdcfe7 from .state import StateStore from .signatures import SignatureStore from .filtering import FilteringStore -- cgit 1.4.1 From 4ffac34a646fa2e763ba9214199d7803d1174959 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 15:02:21 +0000 Subject: Add glob asterisks when running rules. Means that now you can't do exact matches even in override rules, but I think we can live with that. Advantage is that you'll now always get back what was put in to the API. --- synapse/push/__init__.py | 5 +++++ synapse/rest/client/v1/push_rule.py | 5 +---- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index cc05278c8c..6a302305fb 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -137,6 +137,11 @@ class Pusher(object): return False pat = condition['pattern'] + if pat.strip("*?[]") == pat: + # no special glob characters so we assume the user means + # 'contains this string' rather than 'is this string' + pat = "*%s*" % (pat,) + val = _value_for_dotted_key(condition['key'], ev) if val is None: return False diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 0f78fa667c..61e3bc8236 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -98,10 +98,7 @@ class PushRuleRestServlet(ClientV1RestServlet): if 'pattern' not in req_obj: raise InvalidRuleException("Content rule missing 'pattern'") pat = req_obj['pattern'] - if pat.strip("*?[]") == pat: - # no special glob characters so we assume the user means - # 'contains this string' rather than 'is this string' - pat = "*%s*" % (pat,) + conditions = [{ 'kind': 'event_match', 'key': 'content.body', -- cgit 1.4.1 From 017dfaef4c0d4550acd45d27cdcea1c20766684d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 15:52:05 +0000 Subject: Add doc string for __nonzero__ overrides for sync results, raise not implemented if the client attempts to do a gapless sync --- synapse/handlers/sync.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9e07f30b24..dc69b3cfe7 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -49,6 +49,9 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ __slots__ = [] def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if room needs to be part of the sync result. + """ return bool(self.events or self.state or self.ephemeral) @@ -61,6 +64,10 @@ class SyncResult(collections.namedtuple("SyncResult", [ __slots__ = [] def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if the notifier needs to wait for more events when polling for + events. + """ return bool( self.private_user_data or self.public_user_data or self.rooms ) @@ -108,7 +115,7 @@ class SyncHandler(BaseHandler): return self.incremental_sync_with_gap(sync_config, since_token) else: #TODO(mjark): Handle gapless sync - pass + raise NotImplementedError() @defer.inlineCallbacks def initial_sync(self, sync_config): -- cgit 1.4.1 From b4b892f4a3f9194289f57ed2d166eca26da41594 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 15:54:29 +0000 Subject: Spit out server default rules too. --- synapse/push/baserules.py | 11 +++++------ synapse/rest/client/v1/push_rule.py | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 10 deletions(-) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index bd162baade..382de118e0 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -1,9 +1,5 @@ def make_base_rules(user_name): - """ - Nominally we reserve priority class 0 for these rules, although - in practice we just append them to the end so we don't actually need it. - """ - return [ + rules = [ { 'conditions': [ { @@ -46,4 +42,7 @@ def make_base_rules(user_name): } ] } - ] \ No newline at end of file + ] + for r in rules: + r['priority_class'] = 0 + return rules \ No newline at end of file diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 61e3bc8236..faa7919fbb 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -19,6 +19,7 @@ from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, No StoreError from .base import ClientV1RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException +import synapse.push.baserules as baserules import json @@ -26,6 +27,7 @@ import json class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") PRIORITY_CLASS_MAP = { + 'default': 0, 'underride': 1, 'sender': 2, 'room': 3, @@ -137,6 +139,9 @@ class PushRuleRestServlet(ClientV1RestServlet): user, _ = yield self.auth.get_user_by_req(request) + if spec['template'] == 'default': + raise SynapseError(403, "The default rules are immutable.") + content = _parse_json(request) try: @@ -218,6 +223,10 @@ class PushRuleRestServlet(ClientV1RestServlet): # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + for r in rawrules: + r["conditions"] = json.loads(r["conditions"]) + r["actions"] = json.loads(r["actions"]) + rawrules.extend(baserules.make_base_rules(user.to_string())) rules = {'global': {}, 'device': {}} @@ -226,9 +235,6 @@ class PushRuleRestServlet(ClientV1RestServlet): for r in rawrules: rulearray = None - r["conditions"] = json.loads(r["conditions"]) - r["actions"] = json.loads(r["actions"]) - template_name = _priority_class_to_template_name(r['priority_class']) if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: @@ -356,7 +362,9 @@ def _priority_class_to_template_name(pc): def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) - if template_name in ['override', 'underride']: + if template_name in ['default']: + return {k: rule[k] for k in ["conditions", "actions"]} + elif template_name in ['override', 'underride']: return {k: rule[k] for k in ["rule_id", "conditions", "actions"]} elif template_name in ["sender", "room"]: return {k: rule[k] for k in ["rule_id", "actions"]} -- cgit 1.4.1 From 7a1e881665ab72e29ff931f88ee2b65e8e690f74 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 15:56:32 +0000 Subject: Remove debug logging --- synapse/state.py | 9 --------- 1 file changed, 9 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 038e5eba11..8a056ee955 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -214,15 +214,6 @@ class StateHandler(object): if len(v.values()) > 1 } - logger.debug( - "resolve_state_groups Unconflicted state: %s", - unconflicted_state.values(), - ) - logger.debug( - "resolve_state_groups Conflicted state: %s", - conflicted_state.values(), - ) - if event_type: prev_states_events = conflicted_state.get( (event_type, state_key), [] -- cgit 1.4.1 From b724a809c46f782231e18b70a632ce1ea9c540da Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 15:57:53 +0000 Subject: Only auth_events with event if event in event.auth_events --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 35cad4182a..40836e0c51 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -805,7 +805,9 @@ class FederationHandler(BaseHandler): yield self._handle_new_event( origin, ev, auth_events=auth ) - auth_events[(ev.type, ev.state_key)] = ev + + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev except AuthError: pass -- cgit 1.4.1 From 776ac820f9bb7f0e9c2fae9facbee05b0132079e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 15:58:28 +0000 Subject: Briefly doc structure of query_auth API. --- synapse/federation/federation_server.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 84ed0a0ba0..5fbd8b19de 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -232,6 +232,24 @@ class FederationServer(object): @defer.inlineCallbacks def on_query_auth_request(self, origin, content, event_id): + """ + Content is a dict with keys:: + auth_chain (list): A list of events that give the auth chain. + missing (list): A list of event_ids indicating what the other + side (`origin`) think we're missing. + rejects (dict): A mapping from event_id to a 2-tuple of reason + string and a proof (or None) of why the event was rejected. + The keys of this dict give the list of events the `origin` has + rejected. + + Args: + origin (str) + content (dict) + event_id (str) + + Returns: + Deferred: Results in `dict` with the same format as `content` + """ auth_chain = [ (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) for e in content["auth_chain"] -- cgit 1.4.1 From 2cd29dbdd91a8f58f988eb3c417ad6a187fc7e87 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 16:51:58 +0000 Subject: Fix bug where accepting invite over federation didn't work. Add logging. --- synapse/handlers/federation.py | 57 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 40836e0c51..623c54d584 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -343,6 +343,10 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True + + if e.event_id == event.event_id: + continue + try: auth_ids = [e_id for e_id, _ in e.auth_events] auth = { @@ -359,7 +363,9 @@ class FederationHandler(BaseHandler): ) for e in state: - # FIXME: Auth these. + if e.event_id == event.event_id: + continue + e.internal_metadata.outlier = True try: auth_ids = [e_id for e_id, _ in e.auth_events] @@ -376,11 +382,18 @@ class FederationHandler(BaseHandler): e.event_id, ) + auth_ids = [e_id for e_id, _ in event.auth_events] + auth_events = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( target_host, new_event, state=state, current_state=state, + auth_events=auth_events, ) yield self.notifier.on_new_room_event( @@ -752,7 +765,17 @@ class FederationHandler(BaseHandler): origin, event.room_id, event.event_id ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in remote_auth_chain] + ) + for e in remote_auth_chain: + if e.event_id in seen_remotes.keys(): + continue + + if e.event_id == event.event_id: + continue + try: auth_ids = [e_id for e_id, _ in e.auth_events] auth = { @@ -760,10 +783,17 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } e.internal_metadata.outlier = True + + logger.debug( + "do_auth %s missing_auth: %s", + event.event_id, e.event_id + ) yield self._handle_new_event( origin, e, auth_events=auth ) - auth_events[(e.type, e.state_key)] = e + + if e.event_id in event_auth_events: + auth_events[(e.type, e.state_key)] = e except AuthError: pass @@ -788,20 +818,31 @@ class FederationHandler(BaseHandler): local_auth_chain, ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) + # 3. Process any remote auth chain events we haven't seen. - for missing_id in result.get("missing", []): - try: - for e in result["auth_chain"]: - if e.event_id == missing_id: - ev = e - break + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue + if ev.event_id == event.event_id: + continue + + try: auth_ids = [e_id for e_id, _ in ev.auth_events] auth = { (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } ev.internal_metadata.outlier = True + + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) + yield self._handle_new_event( origin, ev, auth_events=auth ) -- cgit 1.4.1 From 0b1cc7cc0bd81f93ed353a6ac67e06f91b62ab90 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 16:24:40 +0000 Subject: Return empty list rather than None when there are no emphemeral events for a room --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index dc69b3cfe7..962686f4bb 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -370,7 +370,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, - ephemeral=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, []) ) logging.debug("Room sync: %r", room_sync) -- cgit 1.4.1 From 4c0da49d7ca58774adad870da98a7e168a5be18c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 22:53:13 +0000 Subject: Resign events when we return them via /query_auth/ --- synapse/handlers/federation.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 623c54d584..8bf5a4cc11 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -741,6 +741,15 @@ class FederationHandler(BaseHandler): local_auth_chain, remote_auth_chain ) + for event in ret["auth_chain"]: + event.signatures.update( + compute_event_signature( + event, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) + logger.debug("on_query_auth reutrning: %s", ret) defer.returnValue(ret) -- cgit 1.4.1 From fe10b882b74479b6d139039956932b98629c8165 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 23:05:49 +0000 Subject: Don't assume all member events have a display nme. --- synapse/push/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 6a302305fb..75867b3c3c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -204,10 +204,11 @@ class Pusher(object): event_type='m.room.member', state_key=ev['user_id'] ) - if len(their_member_events_for_room) > 0: - dn = their_member_events_for_room[0].content['displayname'] - if dn is not None: - ctx['sender_display_name'] = dn + for mev in their_member_events_for_room: + if mev.content['membership'] == 'join' and 'displayname' in mev.content: + dn = mev.content['displayname'] + if dn is not None: + ctx['sender_display_name'] = dn defer.returnValue(ctx) -- cgit 1.4.1 From 0b354fcb8452540baf59b714f4d2186615dc5383 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 23:10:35 +0000 Subject: Again, don't assume all member events have displayname. --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 75867b3c3c..28e5dae81d 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -101,7 +101,7 @@ class Pusher(object): # This loop does two things: # 1) Find our current display name - if mev.state_key == self.user_name: + if mev.state_key == self.user_name and 'displayname' in mev.content: my_display_name = mev.content['displayname'] # and 2) Get the number of people in that room -- cgit 1.4.1 From 365e007bee27e230f1280a2174cfc9abfb2deac2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Sat, 31 Jan 2015 12:48:06 +0000 Subject: Ignore empty strings for display names & room names in notifications --- synapse/push/httppusher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index d4c5f03b01..7c6953c989 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -92,9 +92,9 @@ class HttpPusher(Pusher): if len(ctx['aliases']): d['notification']['room_alias'] = ctx['aliases'][0] - if 'sender_display_name' in ctx: + if 'sender_display_name' in ctx and len(ctx['sender_display_name']) > 0: d['notification']['sender_display_name'] = ctx['sender_display_name'] - if 'name' in ctx: + if 'name' in ctx and len(ctx['name']) > 0: d['notification']['room_name'] = ctx['name'] defer.returnValue(d) -- cgit 1.4.1 From 3b33529dfdf1a94aebd357eebdfc75e903bf1591 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Feb 2015 13:03:25 +0000 Subject: Bump version --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/VERSION b/VERSION index 3b3e723172..492b488cd8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1b +0.6.1c diff --git a/synapse/__init__.py b/synapse/__init__.py index e7e27b06ec..fdfe5eda3e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.1b" +__version__ = "0.6.1c" -- cgit 1.4.1 From d8cf06e525241252bcfac358d44f98d15dba9d0f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 13:18:36 +0000 Subject: Bump version to 0.6.1d --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/VERSION b/VERSION index 492b488cd8..1c29ff4d36 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1c +0.6.1d diff --git a/synapse/__init__.py b/synapse/__init__.py index fdfe5eda3e..dacda89865 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.1c" +__version__ = "0.6.1d" -- cgit 1.4.1 From 22c1ffb0a0b1a817acade3e8a19948b208a2c9f3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 16:02:31 +0000 Subject: Add a media/v1/identicon resource for generating identicons using pydenticon --- setup.py | 1 + synapse/rest/media/v1/identicon_resource.py | 48 +++++++++++++++++++++++++++++ synapse/rest/media/v1/media_repository.py | 2 ++ 3 files changed, 51 insertions(+) create mode 100644 synapse/rest/media/v1/identicon_resource.py (limited to 'synapse') diff --git a/setup.py b/setup.py index 28e8188f94..bd2766b24c 100755 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ setup( "py-bcrypt", "frozendict>=0.4", "pillow", + "pydenticon", ], dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", diff --git a/synapse/rest/media/v1/identicon_resource.py b/synapse/rest/media/v1/identicon_resource.py new file mode 100644 index 0000000000..8878d4c3d4 --- /dev/null +++ b/synapse/rest/media/v1/identicon_resource.py @@ -0,0 +1,48 @@ +from pydenticon import Generator +from twisted.web.resource import Resource + +FOREGROUND = [ + "rgb(45,79,255)", + "rgb(254,180,44)", + "rgb(226,121,234)", + "rgb(30,179,253)", + "rgb(232,77,65)", + "rgb(49,203,115)", + "rgb(141,69,170)" +] + +BACKGROUND = "rgb(224,224,224)" +SIZE = 5 + + +class IdenticonResource(Resource): + isLeaf = True + + def __init__(self): + Resource.__init__(self) + self.generator = Generator( + SIZE, SIZE, foreground=FOREGROUND, background=BACKGROUND, + ) + + def generate_identicon(self, name, width, height): + v_padding = width % SIZE + h_padding = height % SIZE + top_padding = v_padding // 2 + left_padding = h_padding // 2 + bottom_padding = v_padding - top_padding + right_padding = h_padding - left_padding + width -= v_padding + height -= h_padding + padding = (top_padding, bottom_padding, left_padding, right_padding) + identicon = self.generator.generate( + name, width, height, padding=padding + ) + return identicon + + def render_GET(self, request): + name = "/".join(request.postpath) + width = int(request.args.get("width", 96)) + height = int(request.args.get("width", 96)) + identicon_bytes = self.generate_identicon(name, width, height) + request.setHeader(b"Content-Type", b"image/png") + return identicon_bytes diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 461cc001f1..61ed90f39f 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -16,6 +16,7 @@ from .upload_resource import UploadResource from .download_resource import DownloadResource from .thumbnail_resource import ThumbnailResource +from .identicon_resource import IdenticonResource from .filepath import MediaFilePaths from twisted.web.resource import Resource @@ -75,3 +76,4 @@ class MediaRepositoryResource(Resource): self.putChild("upload", UploadResource(hs, filepaths)) self.putChild("download", DownloadResource(hs, filepaths)) self.putChild("thumbnail", ThumbnailResource(hs, filepaths)) + self.putChild("identicon", IdenticonResource()) -- cgit 1.4.1 From a006d168c556ca71ad0bbb680c60f1ba170338fb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Feb 2015 16:05:34 +0000 Subject: Actually merge into develop. --- synapse/storage/__init__.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9bbd553dfc..1f207495f6 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,12 +67,9 @@ SCHEMAS = [ "event_signatures", "pusher", "media_repository", -<<<<<<< HEAD "application_services" -======= "filtering", "rejections", ->>>>>>> develop ] @@ -92,17 +89,13 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, -<<<<<<< HEAD - EventFederationStore, MediaRepositoryStore, - ApplicationServiceStore -======= + ApplicationServiceStore, EventFederationStore, MediaRepositoryStore, RejectionsStore, FilteringStore, PusherStore, PushRuleStore ->>>>>>> develop ): def __init__(self, hs): -- cgit 1.4.1 From 038f5afb07aa1df27528116eeed0f6e44d67c579 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 16:29:18 +0000 Subject: Spell height more correctly --- synapse/rest/media/v1/identicon_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/media/v1/identicon_resource.py b/synapse/rest/media/v1/identicon_resource.py index 8878d4c3d4..636e3a33c2 100644 --- a/synapse/rest/media/v1/identicon_resource.py +++ b/synapse/rest/media/v1/identicon_resource.py @@ -42,7 +42,7 @@ class IdenticonResource(Resource): def render_GET(self, request): name = "/".join(request.postpath) width = int(request.args.get("width", 96)) - height = int(request.args.get("width", 96)) + height = int(request.args.get("height", 96)) identicon_bytes = self.generate_identicon(name, width, height) request.setHeader(b"Content-Type", b"image/png") return identicon_bytes -- cgit 1.4.1 From f2eda123b78a7b61587db60d6c8995571a7eb7fb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 16:32:33 +0000 Subject: Fix setting identicon width and height --- synapse/rest/media/v1/identicon_resource.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/media/v1/identicon_resource.py b/synapse/rest/media/v1/identicon_resource.py index 636e3a33c2..12d98e1774 100644 --- a/synapse/rest/media/v1/identicon_resource.py +++ b/synapse/rest/media/v1/identicon_resource.py @@ -41,8 +41,8 @@ class IdenticonResource(Resource): def render_GET(self, request): name = "/".join(request.postpath) - width = int(request.args.get("width", 96)) - height = int(request.args.get("height", 96)) + width = int(request.args.get("width", [96])[0]) + height = int(request.args.get("height", [96])[0]) identicon_bytes = self.generate_identicon(name, width, height) request.setHeader(b"Content-Type", b"image/png") return identicon_bytes -- cgit 1.4.1 From 941f59101b51e9225dbdc38b22110a01de194242 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Feb 2015 16:56:01 +0000 Subject: Don't fail an entire request if one of the returned events fails a signature check. If an event does fail a signature check, look in the local database and request it from the originator. --- synapse/federation/federation_client.py | 107 ++++++++++++++++++++++++-------- synapse/storage/__init__.py | 21 ++++--- 2 files changed, 94 insertions(+), 34 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index e1539bd0e0..b809e935a0 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -224,17 +224,17 @@ class FederationClient(object): for p in result.get("auth_chain", []) ] - for i, pdu in enumerate(pdus): - pdus[i] = yield self._check_sigs_and_hash(pdu) - - # FIXME: We should handle signature failures more gracefully. + signed_pdus = yield self._check_sigs_and_hash_and_fetch( + pdus, outlier=True + ) - for i, pdu in enumerate(auth_chain): - auth_chain[i] = yield self._check_sigs_and_hash(pdu) + signed_auth = yield self._check_sigs_and_hash_and_fetch( + auth_chain, outlier=True + ) - # FIXME: We should handle signature failures more gracefully. + signed_auth.sort(key=lambda e: e.depth) - defer.returnValue((pdus, auth_chain)) + defer.returnValue((signed_pdus, signed_auth)) @defer.inlineCallbacks @log_function @@ -248,14 +248,13 @@ class FederationClient(object): for p in res["auth_chain"] ] - for i, pdu in enumerate(auth_chain): - auth_chain[i] = yield self._check_sigs_and_hash(pdu) - - # FIXME: We should handle signature failures more gracefully. + signed_auth = yield self._check_sigs_and_hash_and_fetch( + auth_chain, outlier=True + ) - auth_chain.sort(key=lambda e: e.depth) + signed_auth.sort(key=lambda e: e.depth) - defer.returnValue(auth_chain) + defer.returnValue(signed_auth) @defer.inlineCallbacks def make_join(self, destination, room_id, user_id): @@ -291,21 +290,19 @@ class FederationClient(object): for p in content.get("auth_chain", []) ] - for i, pdu in enumerate(state): - state[i] = yield self._check_sigs_and_hash(pdu) - - # FIXME: We should handle signature failures more gracefully. - - for i, pdu in enumerate(auth_chain): - auth_chain[i] = yield self._check_sigs_and_hash(pdu) + signed_state = yield self._check_sigs_and_hash_and_fetch( + state, outlier=True + ) - # FIXME: We should handle signature failures more gracefully. + signed_auth = yield self._check_sigs_and_hash_and_fetch( + auth_chain, outlier=True + ) auth_chain.sort(key=lambda e: e.depth) defer.returnValue({ - "state": state, - "auth_chain": auth_chain, + "state": signed_state, + "auth_chain": signed_auth, }) @defer.inlineCallbacks @@ -353,12 +350,18 @@ class FederationClient(object): ) auth_chain = [ - (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + self.event_from_pdu_json(e) for e in content["auth_chain"] ] + signed_auth = yield self._check_sigs_and_hash_and_fetch( + auth_chain, outlier=True + ) + + signed_auth.sort(key=lambda e: e.depth) + ret = { - "auth_chain": auth_chain, + "auth_chain": signed_auth, "rejects": content.get("rejects", []), "missing": content.get("missing", []), } @@ -374,6 +377,58 @@ class FederationClient(object): return event + @defer.inlineCallbacks + def _check_sigs_and_hash_and_fetch(self, pdus, outlier=False): + """Takes a list of PDUs and checks the signatures and hashs of each + one. If a PDU fails its signature check then we check if we have it in + the database and if not then request if from the originating server of + that PDU. + + If a PDU fails its content hash check then it is redacted. + + The given list of PDUs are not modified, instead the function returns + a new list. + + Args: + pdu (list) + outlier (bool) + + Returns: + Deferred : A list of PDUs that have valid signatures and hashes. + """ + signed_pdus = [] + for pdu in pdus: + try: + new_pdu = yield self._check_sigs_and_hash(pdu) + signed_pdus.append(new_pdu) + except SynapseError: + # FIXME: We should handle signature failures more gracefully. + + # Check local db. + new_pdu = yield self.store.get_event( + pdu.event_id, + allow_rejected=True + ) + if new_pdu: + signed_pdus.append(new_pdu) + continue + + # Check pdu.origin + new_pdu = yield self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + ) + + if new_pdu: + signed_pdus.append(new_pdu) + continue + + logger.warn("Failed to find copy of %s with valid signature") + + defer.returnValue(signed_pdus) + + @defer.inlineCallbacks def _check_sigs_and_hash(self, pdu): """Throws a SynapseError if the PDU does not have the correct diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7c54b1b9d3..b4a7a3f068 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -128,16 +128,21 @@ class DataStore(RoomMemberStore, RoomStore, pass @defer.inlineCallbacks - def get_event(self, event_id, allow_none=False): - events = yield self._get_events([event_id]) + def get_event(self, event_id, check_redacted=True, + get_prev_content=False, allow_rejected=False, + allow_none=False): + event = yield self.runInteraction( + "get_event", self._get_event_txn, + event_id, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) - if not events: - if allow_none: - defer.returnValue(None) - else: - raise RuntimeError("Could not find event %s" % (event_id,)) + if not event and not allow_none: + raise RuntimeError("Could not find event %s" % (event_id,)) - defer.returnValue(events[0]) + defer.returnValue(event) @log_function def _persist_event_txn(self, txn, event, context, backfilled, -- cgit 1.4.1 From 1bb0528316729eb19536d8f3a45564d8dc02cb4a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 16:34:07 +0000 Subject: Add Cache-Control header to identicon --- synapse/rest/media/v1/identicon_resource.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'synapse') diff --git a/synapse/rest/media/v1/identicon_resource.py b/synapse/rest/media/v1/identicon_resource.py index 12d98e1774..912856386a 100644 --- a/synapse/rest/media/v1/identicon_resource.py +++ b/synapse/rest/media/v1/identicon_resource.py @@ -45,4 +45,7 @@ class IdenticonResource(Resource): height = int(request.args.get("height", [96])[0]) identicon_bytes = self.generate_identicon(name, width, height) request.setHeader(b"Content-Type", b"image/png") + request.setHeader( + b"Cache-Control", b"public,max-age=86400,s-maxage=86400" + ) return identicon_bytes -- cgit 1.4.1 From 40c6fe1b81e4d92cba797b0c966fd774e2a60a28 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Feb 2015 17:06:37 +0000 Subject: Don't bother requesting PDUs with bad signatures from the same server --- synapse/federation/federation_client.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index b809e935a0..f87e84db73 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -225,11 +225,11 @@ class FederationClient(object): ] signed_pdus = yield self._check_sigs_and_hash_and_fetch( - pdus, outlier=True + destination, pdus, outlier=True ) signed_auth = yield self._check_sigs_and_hash_and_fetch( - auth_chain, outlier=True + destination, auth_chain, outlier=True ) signed_auth.sort(key=lambda e: e.depth) @@ -249,7 +249,7 @@ class FederationClient(object): ] signed_auth = yield self._check_sigs_and_hash_and_fetch( - auth_chain, outlier=True + destination, auth_chain, outlier=True ) signed_auth.sort(key=lambda e: e.depth) @@ -291,11 +291,11 @@ class FederationClient(object): ] signed_state = yield self._check_sigs_and_hash_and_fetch( - state, outlier=True + destination, state, outlier=True ) signed_auth = yield self._check_sigs_and_hash_and_fetch( - auth_chain, outlier=True + destination, auth_chain, outlier=True ) auth_chain.sort(key=lambda e: e.depth) @@ -355,7 +355,7 @@ class FederationClient(object): ] signed_auth = yield self._check_sigs_and_hash_and_fetch( - auth_chain, outlier=True + destination, auth_chain, outlier=True ) signed_auth.sort(key=lambda e: e.depth) @@ -378,7 +378,7 @@ class FederationClient(object): return event @defer.inlineCallbacks - def _check_sigs_and_hash_and_fetch(self, pdus, outlier=False): + def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False): """Takes a list of PDUs and checks the signatures and hashs of each one. If a PDU fails its signature check then we check if we have it in the database and if not then request if from the originating server of @@ -414,15 +414,16 @@ class FederationClient(object): continue # Check pdu.origin - new_pdu = yield self.get_pdu( - destinations=[pdu.origin], - event_id=pdu.event_id, - outlier=outlier, - ) - - if new_pdu: - signed_pdus.append(new_pdu) - continue + if pdu.origin != origin: + new_pdu = yield self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + ) + + if new_pdu: + signed_pdus.append(new_pdu) + continue logger.warn("Failed to find copy of %s with valid signature") -- cgit 1.4.1 From 4574b5a9e6a52086bd08d23409cc6e15070fe904 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 17:22:40 +0000 Subject: Generate a list of dependencies from synapse/python_dependencies.py --- synapse/python_dependencies.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 826a36f203..da15c6da45 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -18,6 +18,27 @@ REQUIREMENTS = { "pillow": ["PIL"], } +def github_link(project, version, egg): + return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) + +DEPENDENCY_LINKS=[ + github_link( + project="matrix-org/syutil", + version="v0.0.2", + egg="syutil-0.0.2", + ), + github_link( + project="matrix-org/matrix-angular-sdk", + version="v0.6.0", + egg="matrix_angular_sdk-0.6.0", + ), + github_link( + project="pyca/pynacl", + version="d4d3175589b892f6ea7c22f466e0e223853516fa", + egg="pynacl-0.3.0", + ) +] + class MissingRequirementError(Exception): pass @@ -78,3 +99,23 @@ def check_requirements(): "Unexpected version of %r in %r. %r != %r" % (dependency, file_path, version, required_version) ) + +def list_requirements(): + result = [] + linked = [] + for link in DEPENDENCY_LINKS: + egg = link.split("#egg=")[1] + linked.append(egg.split('-')[0]) + result.append(link) + for requirement in REQUIREMENTS: + is_linked = False + for link in linked: + if requirement.replace('-','_').startswith(link): + is_linked = True + if not is_linked: + result.append(requirement) + return result + +if __name__ == "__main__": + import sys + sys.stdout.writelines(req + "\n" for req in list_requirements()) -- cgit 1.4.1 From a2da04b8ab269a8353ecfca20132c273e972a4db Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 17:37:26 +0000 Subject: Add pydenticon to python_dependencies --- synapse/python_dependencies.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 826a36f203..168fab0658 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -16,6 +16,7 @@ REQUIREMENTS = { "py-bcrypt": ["bcrypt"], "frozendict>=0.4": ["frozendict"], "pillow": ["PIL"], + "pydenticon": ["pydenticon"], } -- cgit 1.4.1 From 1a2de0c5feb1183b35045bb7fb9e379a9598d1cb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Feb 2015 17:39:41 +0000 Subject: Implement txns for AS (un)registration. --- synapse/handlers/appservice.py | 3 +- synapse/storage/__init__.py | 2 +- synapse/storage/appservice.py | 120 +++++++++++++++++++++++++++++++++-------- 3 files changed, 102 insertions(+), 23 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8bd475cbfd..da994ba8e0 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -43,7 +43,8 @@ class ApplicationServicesHandler(BaseHandler): "Consult the home server admin.", errcode=Codes.FORBIDDEN ) - # TODO store this AS + logger.info("Updating application service info...") + yield self.store.update_app_service(app_service) def unregister(self, token): logger.info("Unregister as_token=%s", token) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1f207495f6..6ff0093136 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,7 +67,7 @@ SCHEMAS = [ "event_signatures", "pusher", "media_repository", - "application_services" + "application_services", "filtering", "rejections", ] diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 5a0e47e0d4..db0c546211 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -15,11 +15,17 @@ import logging from twisted.internet import defer +from synapse.api.errors import StoreError from ._base import SQLBaseStore logger = logging.getLogger(__name__) +namespace_enum = [ + "users", # 0 + "aliases", # 1 + "rooms" # 2 +] # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): @@ -30,25 +36,26 @@ class ApplicationService(object): def __init__(self, token, url=None, namespaces=None): self.token = token - if url: - self.url = url - if namespaces: - self._set_namespaces(namespaces) + self.url = url + self.namespaces = self._get_namespaces(namespaces) - def _set_namespaces(self, namespaces): + def _get_namespaces(self, namespaces): # Sanity check that it is of the form: # { # users: ["regex",...], # aliases: ["regex",...], # rooms: ["regex",...], # } + if not namespaces: + return None + for ns in ["users", "rooms", "aliases"]: if type(namespaces[ns]) != list: raise ValueError("Bad namespace value for '%s'", ns) for regex in namespaces[ns]: if not isinstance(regex, basestring): raise ValueError("Expected string regex for ns '%s'", ns) - self.namespaces = namespaces + return namespaces def is_interested(self, event): """Check if this service is interested in this event. @@ -110,10 +117,38 @@ class ApplicationServiceStore(SQLBaseStore): This removes all AS specific regex and the base URL. The token is the only thing preserved for future registration attempts. """ - # TODO: DELETE FROM application_services_regex WHERE id=this service - # TODO: SET url=NULL WHERE token=token - # TODO: Update cache - pass + yield self.runInteraction( + "unregister_app_service", + self._unregister_app_service_txn, + token, + ) + # update cache TODO: Should this be in the txn? + for service in self.cache.services: + if service.token == token: + service.url = None + service.namespaces = None + + def _unregister_app_service_txn(self, txn, token): + # kill the url to prevent pushes + txn.execute( + "UPDATE application_services SET url=NULL WHERE token=?", + (token,) + ) + + # cleanup regex + as_id = self._get_as_id_txn(txn, token) + if not as_id: + logger.warning( + "unregister_app_service_txn: Failed to find as_id for token=", + token + ) + return False + + txn.execute( + "DELETE FROM application_services_regex WHERE as_id=?", + (as_id,) + ) + return True def update_app_service(self, service): """Update an application service, clobbering what was previously there. @@ -124,12 +159,61 @@ class ApplicationServiceStore(SQLBaseStore): # NB: There is no "insert" since we provide no public-facing API to # allocate new ASes. It relies on the server admin inserting the AS # token into the database manually. + if not service.token or not service.url: + raise StoreError(400, "Token and url must be specified.") + + yield self.runInteraction( + "update_app_service", + self._update_app_service_txn, + service + ) + + # update cache TODO: Should this be in the txn? + for (index, cache_service) in enumerate(self.cache.services): + if service.token == cache_service.token: + self.cache.services[index] = service + logger.info("Updated: %s", service) + return + # new entry + self.cache.services.append(service) + logger.info("Updated(new): %s", service) + + def _update_app_service_txn(self, txn, service): + as_id = self._get_as_id_txn(txn, service.token) + if not as_id: + logger.warning( + "update_app_service_txn: Failed to find as_id for token=", + service.token + ) + return False + + txn.execute( + "UPDATE application_services SET url=? WHERE id=?", + (service.url, as_id,) + ) + # cleanup regex + txn.execute( + "DELETE FROM application_services_regex WHERE id=?", + (as_id,) + ) + for (ns_int, ns_str) in enumerate(namespace_enum): + if ns_str in service.namespaces: + for regex in service.namespaces[ns_str]: + txn.execute( + "INSERT INTO application_services_regex(" + "as_id, namespace, regex) values(?,?,?)", + (as_id, ns_int, regex) + ) + return True - # TODO: UPDATE application_services, SET url WHERE token=service.token - # TODO: DELETE FROM application_services_regex WHERE id=this service - # TODO: INSERT INTO application_services_regex - # TODO: Update cache - pass + def _get_as_id_txn(self, txn, token): + cursor = txn.execute( + "SELECT id FROM application_services WHERE token=?", + (token,) + ) + res = cursor.fetchone() + if res: + return res[0] def get_services_for_event(self, event): return self.cache.get_services_for_event(event) @@ -161,12 +245,6 @@ class ApplicationServiceStore(SQLBaseStore): sql = ("SELECT * FROM application_services LEFT JOIN " "application_services_regex ON application_services.id = " "application_services_regex.as_id") - - namespace_enum = [ - "users", # 0 - "aliases", # 1 - "rooms" # 2 - ] # SQL results in the form: # [ # { -- cgit 1.4.1 From 9ff349a3cb1868bb2827047ed0f0d01a9a4c38c7 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Feb 2015 17:42:49 +0000 Subject: Add defers in the right places. --- synapse/storage/appservice.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index db0c546211..dd9b349370 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -111,6 +111,7 @@ class ApplicationServiceStore(SQLBaseStore): self.cache = ApplicationServiceCache() self._populate_cache() + @defer.inlineCallbacks def unregister_app_service(self, token): """Unregisters this service. @@ -150,6 +151,7 @@ class ApplicationServiceStore(SQLBaseStore): ) return True + @defer.inlineCallbacks def update_app_service(self, service): """Update an application service, clobbering what was previously there. -- cgit 1.4.1 From e7ca813dd476c83497d4130ad8efa9424d86e921 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 10:38:14 +0000 Subject: Try to ensure we don't persist an event we have already persisted. In persist_event check if we already have the event, if so then update instead of replacing so that we don't cause a bump of the stream_ordering. --- synapse/handlers/federation.py | 42 ++++++++++++++++++++++++++------------- synapse/storage/__init__.py | 40 +++++++++++++++++++++++++++++++++---- tests/handlers/test_federation.py | 5 ++++- 3 files changed, 68 insertions(+), 19 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8bf5a4cc11..c384789c2f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -112,6 +112,14 @@ class FederationHandler(BaseHandler): logger.debug("Event: %s", event) + event_ids = set() + if state: + event_ids += {e.event_id for e in state} + if auth_chain: + event_ids += {e.event_id for e in auth_chain} + + seen_ids = (yield self.store.have_events(event_ids)).keys() + # FIXME (erikj): Awful hack to make the case where we are not currently # in the room work current_state = None @@ -124,20 +132,26 @@ class FederationHandler(BaseHandler): current_state = state if state and auth_chain is not None: - for e in state: - e.internal_metadata.outlier = True - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - yield self._handle_new_event(origin, e, auth_events=auth) - except: - logger.exception( - "Failed to handle state event %s", - e.event_id, - ) + for list_of_pdus in [auth_chain, state]: + for e in list_of_pdus: + if e.event_id in seen_ids: + continue + + e.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + except: + logger.exception( + "Failed to handle state event %s", + e.event_id, + ) try: yield self._handle_new_event( diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b4a7a3f068..93aefe0c48 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -161,6 +161,39 @@ class DataStore(RoomMemberStore, RoomStore, outlier = event.internal_metadata.is_outlier() + have_persisted = self._simple_select_one_onecol_txn( + txn, + table="event_json", + keyvalues={"event_id": event.event_id}, + retcol="event_id", + allow_none=True, + ) + + metadata_json = encode_canonical_json( + event.internal_metadata.get_dict() + ) + + if have_persisted: + if not outlier: + sql = ( + "UPDATE event_json SET internal_metadata = ?" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (metadata_json.decode("UTF-8"), event.event_id,) + ) + + sql = ( + "UPDATE events SET outlier = 0" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (event.event_id,) + ) + return + event_dict = { k: v for k, v in event.get_dict().items() @@ -170,10 +203,6 @@ class DataStore(RoomMemberStore, RoomStore, ] } - metadata_json = encode_canonical_json( - event.internal_metadata.get_dict() - ) - self._simple_insert_txn( txn, table="event_json", @@ -482,6 +511,9 @@ class DataStore(RoomMemberStore, RoomStore, the rejected reason string if we rejected the event, else maps to None. """ + if not event_ids: + return defer.succeed({}) + def f(txn): sql = ( "SELECT e.event_id, reason FROM events as e " diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 44dbce6bea..4270481139 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -91,7 +91,10 @@ class FederationTestCase(unittest.TestCase): self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) - self.datastore.have_events.return_value = defer.succeed({}) + + def have_events(event_ids): + return defer.succeed({}) + self.datastore.have_events.side_effect = have_events def annotate(ev, old_state=None): context = Mock() -- cgit 1.4.1 From 51969f9e5f4774e4d0ddcc2e8ebcf96803cbf295 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 10:40:14 +0000 Subject: Return rejected events if asked for it over federation. --- synapse/handlers/federation.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c384789c2f..0161fbe49c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -632,6 +632,7 @@ class FederationHandler(BaseHandler): event = yield self.store.get_event( event_id, allow_none=True, + allow_rejected=True, ) if event: -- cgit 1.4.1 From 0f48e22ef66ff8a34d4af13c25e20a461c8a8390 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 10:43:29 +0000 Subject: PEP8 --- synapse/federation/federation_client.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index f87e84db73..9ceb66e6f4 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -429,7 +429,6 @@ class FederationClient(object): defer.returnValue(signed_pdus) - @defer.inlineCallbacks def _check_sigs_and_hash(self, pdu): """Throws a SynapseError if the PDU does not have the correct -- cgit 1.4.1 From 4ff2273b300c0a40fcf054a613aac24c51af22f1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 11:23:26 +0000 Subject: Add FIXME note. --- synapse/handlers/federation.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0161fbe49c..322c1b407d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -716,6 +716,8 @@ class FederationHandler(BaseHandler): context.rejected = RejectedReason.AUTH_ERROR + # FIXME: Don't store as rejected with AUTH_ERROR if we haven't + # seen all the auth events. yield self.store.persist_event( event, context=context, -- cgit 1.4.1 From 06c34bfbaee553b851e0f02d8e17d5bff7360376 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 11:23:44 +0000 Subject: Give exception better message --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 322c1b407d..9d7557f578 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1000,7 +1000,7 @@ class FederationHandler(BaseHandler): if reason is None: # FIXME: ERRR?! logger.warn("Could not find reason for %s", e.event_id) - raise RuntimeError("") + raise RuntimeError("Could not find reason for %s" % e.event_id) reason_map[e.event_id] = reason -- cgit 1.4.1 From 197f3ea4bad066da251c7925336baab8bee296c9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 11:26:33 +0000 Subject: Implement regex checks for app services. Expose handler.get_services_for_event which manages the checks for all services. --- synapse/handlers/appservice.py | 25 +++++++++++++-- synapse/storage/appservice.py | 71 ++++++++++++++++++++++++++---------------- 2 files changed, 67 insertions(+), 29 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index da994ba8e0..9b8dd1bb49 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -34,9 +34,11 @@ class ApplicationServicesHandler(BaseHandler): logger.info("Register -> %s", app_service) # check the token is recognised try: - stored_service = yield self.store.get_app_service(app_service.token) + stored_service = yield self.store.get_app_service_by_token( + app_service.token + ) if not stored_service: - raise StoreError(404, "Application Service Not found") + raise StoreError(404, "Application service not found") except StoreError: raise SynapseError( 403, "Unrecognised application services token. " @@ -50,6 +52,25 @@ class ApplicationServicesHandler(BaseHandler): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) + def get_services_for_event(self, event): + """Retrieve a list of application services interested in this event. + + Args: + event(Event): The event to check. + Returns: + list: A list of services interested in this + event based on the service regex. + """ + # We need to know the aliases associated with this event.room_id, if any + alias_list = [] # TODO + + interested_list = [ + s for s in self.store.get_app_services() if ( + s.is_interested(event, alias_list) + ) + ] + return interested_list + def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index dd9b349370..c4e50be4c6 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import re from twisted.internet import defer +from synapse.api.constants import EventTypes from synapse.api.errors import StoreError from ._base import SQLBaseStore @@ -27,6 +29,7 @@ namespace_enum = [ "rooms" # 2 ] + # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): """Defines an application service. @@ -37,9 +40,9 @@ class ApplicationService(object): def __init__(self, token, url=None, namespaces=None): self.token = token self.url = url - self.namespaces = self._get_namespaces(namespaces) + self.namespaces = self._check_namespaces(namespaces) - def _get_namespaces(self, namespaces): + def _check_namespaces(self, namespaces): # Sanity check that it is of the form: # { # users: ["regex",...], @@ -57,22 +60,50 @@ class ApplicationService(object): raise ValueError("Expected string regex for ns '%s'", ns) return namespaces - def is_interested(self, event): + def _matches_regex(self, test_string, namespace_key): + for regex in self.namespaces[namespace_key]: + if re.match(regex, test_string): + return True + return False + + def _matches_user(self, event): + if (hasattr(event, "user_id") and + self._matches_regex(event.user_id, "users")): + return True + # also check m.room.member state key + if (hasattr(event, "type") and event.type == EventTypes.Member + and hasattr(event, "state_key") + and self._matches_regex(event.state_key, "users")): + return True + return False + + def _matches_room_id(self, event): + if hasattr(event, "room_id"): + return self._matches_regex(event.room_id, "rooms") + return False + + def _matches_aliases(self, event, alias_list): + for alias in alias_list: + if self._matches_regex(alias, "aliases"): + return True + return False + + def is_interested(self, event, aliases_for_event=None): """Check if this service is interested in this event. Args: event(Event): The event to check. + aliases_for_event(list): A list of all the known room aliases for + this event. Returns: bool: True if this service would like to know about this event. """ - # NB: This does not check room alias regex matches because that requires - # more context that an Event can provide. Room alias matches are checked - # in the ApplicationServiceHandler. - - # TODO check if event.room_id regex matches - # TODO check if event.user_id regex matches (or m.room.member state_key) + if aliases_for_event is None: + aliases_for_event = [] - return True + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) def __str__(self): return "ApplicationService: %s" % (self.__dict__,) @@ -89,20 +120,6 @@ class ApplicationServiceCache(object): def __init__(self): self.services = [] - def get_services_for_event(self, event): - """Retrieve a list of application services interested in this event. - - Args: - event(Event): The event to check. - Returns: - list: A list of services interested in this - event based on the service regex. - """ - interested_list = [ - s for s in self.services if s.is_event_claimed(event) - ] - return interested_list - class ApplicationServiceStore(SQLBaseStore): @@ -217,10 +234,10 @@ class ApplicationServiceStore(SQLBaseStore): if res: return res[0] - def get_services_for_event(self, event): - return self.cache.get_services_for_event(event) + def get_app_services(self): + return self.cache.services - def get_app_service(self, token, from_cache=True): + def get_app_service_by_token(self, token, from_cache=True): """Get the application service with the given token. Args: -- cgit 1.4.1 From 3bd2841fdbbbc2e290d13cbd1aa9becc315d2f1c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 11:37:52 +0000 Subject: Everyone loves SQL typos --- synapse/storage/appservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index c4e50be4c6..07ed0adcf8 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -212,7 +212,7 @@ class ApplicationServiceStore(SQLBaseStore): ) # cleanup regex txn.execute( - "DELETE FROM application_services_regex WHERE id=?", + "DELETE FROM application_services_regex WHERE as_id=?", (as_id,) ) for (ns_int, ns_str) in enumerate(namespace_enum): -- cgit 1.4.1 From a060b47b13037da56ed8db2978a297133c23fc7f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 13:17:28 +0000 Subject: Add namespace constants. Add restrict_to option to limit namespace checks. --- synapse/handlers/appservice.py | 25 +++++++++++++++---------- synapse/storage/appservice.py | 41 ++++++++++++++++++++++++----------------- 2 files changed, 39 insertions(+), 27 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 9b8dd1bb49..bf68b33398 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -52,11 +52,12 @@ class ApplicationServicesHandler(BaseHandler): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) - def get_services_for_event(self, event): + def get_services_for_event(self, event, restrict_to=""): """Retrieve a list of application services interested in this event. Args: event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. Returns: list: A list of services interested in this event based on the service regex. @@ -66,7 +67,7 @@ class ApplicationServicesHandler(BaseHandler): interested_list = [ s for s in self.store.get_app_services() if ( - s.is_interested(event, alias_list) + s.is_interested(event, restrict_to, alias_list) ) ] return interested_list @@ -80,11 +81,15 @@ class ApplicationServicesHandler(BaseHandler): Args: event(Event): The event to push out to interested services. """ - # TODO: Gather interested services - # get_services_for_event(event) <-- room IDs and user IDs - # Get a list of room aliases. Check regex. - # TODO: If unknown user: poke User Query API. - # TODO: If unknown room alias: poke Room Alias Query API. - - # TODO: Fork off pushes to these services - XXX First cut, best effort - pass + # Gather interested services + services = self.get_services_for_event(event) + if len(services) == 0: + return # no services need notifying + + # Do we know this user exists? If not, poke the user query API for + # all services which match that user regex. + + # Do we know this room alias exists? If not, poke the room alias query + # API for all services which match that room alias regex. + + # Fork off pushes to these services - XXX First cut, best effort diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 07ed0adcf8..277741fced 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -23,12 +23,6 @@ from ._base import SQLBaseStore logger = logging.getLogger(__name__) -namespace_enum = [ - "users", # 0 - "aliases", # 1 - "rooms" # 2 -] - # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): @@ -36,6 +30,10 @@ class ApplicationService(object): Provides methods to check if this service is "interested" in events. """ + NS_USERS = "users" + NS_ALIASES = "aliases" + NS_ROOMS = "rooms" + NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None): self.token = token @@ -52,7 +50,7 @@ class ApplicationService(object): if not namespaces: return None - for ns in ["users", "rooms", "aliases"]: + for ns in ApplicationService.NS_LIST: if type(namespaces[ns]) != list: raise ValueError("Bad namespace value for '%s'", ns) for regex in namespaces[ns]: @@ -68,31 +66,36 @@ class ApplicationService(object): def _matches_user(self, event): if (hasattr(event, "user_id") and - self._matches_regex(event.user_id, "users")): + self._matches_regex( + event.user_id, ApplicationService.NS_USERS)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member and hasattr(event, "state_key") - and self._matches_regex(event.state_key, "users")): + and self._matches_regex( + event.state_key, ApplicationService.NS_USERS)): return True return False def _matches_room_id(self, event): if hasattr(event, "room_id"): - return self._matches_regex(event.room_id, "rooms") + return self._matches_regex( + event.room_id, ApplicationService.NS_ROOMS + ) return False def _matches_aliases(self, event, alias_list): for alias in alias_list: - if self._matches_regex(alias, "aliases"): + if self._matches_regex(alias, ApplicationService.NS_ALIASES): return True return False - def is_interested(self, event, aliases_for_event=None): + def is_interested(self, event, restrict_to=None, aliases_for_event=None): """Check if this service is interested in this event. Args: event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. aliases_for_event(list): A list of all the known room aliases for this event. Returns: @@ -100,6 +103,9 @@ class ApplicationService(object): """ if aliases_for_event is None: aliases_for_event = [] + if restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so raise a general exception + raise Exception("Unexpected restrict_to value: %s". restrict_to) return (self._matches_user(event) or self._matches_aliases(event, aliases_for_event) @@ -215,7 +221,7 @@ class ApplicationServiceStore(SQLBaseStore): "DELETE FROM application_services_regex WHERE as_id=?", (as_id,) ) - for (ns_int, ns_str) in enumerate(namespace_enum): + for (ns_int, ns_str) in enumerate(ApplicationService.NS_LIST): if ns_str in service.namespaces: for regex in service.namespaces[ns_str]: txn.execute( @@ -285,9 +291,9 @@ class ApplicationServiceStore(SQLBaseStore): "url": res["url"], "token": as_token, "namespaces": { - "users": [], - "aliases": [], - "rooms": [] + ApplicationService.NS_USERS: [], + ApplicationService.NS_ALIASES: [], + ApplicationService.NS_ROOMS: [] } } # add the namespace regex if one exists @@ -295,7 +301,8 @@ class ApplicationServiceStore(SQLBaseStore): if ns_int is None: continue try: - services[as_token]["namespaces"][namespace_enum[ns_int]].append( + services[as_token]["namespaces"][ + ApplicationService.NS_LIST[ns_int]].append( res["regex"] ) except IndexError: -- cgit 1.4.1 From fed29251d70b050ea5799708b6b13be9617d1f6d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 13:23:58 +0000 Subject: Spelling --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9d7557f578..b9b2e25d1f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -767,7 +767,7 @@ class FederationHandler(BaseHandler): ) ) - logger.debug("on_query_auth reutrning: %s", ret) + logger.debug("on_query_auth returning: %s", ret) defer.returnValue(ret) -- cgit 1.4.1 From f2c039bfb958ed349bce42098e296995786374cc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 13:29:27 +0000 Subject: Implement restricted namespace checks. Begin fleshing out the main hook for notifying application services. --- synapse/handlers/appservice.py | 19 +++++++++++++++++++ synapse/storage/appservice.py | 21 ++++++++++++++++----- 2 files changed, 35 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index bf68b33398..dac63e2245 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -17,6 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError +from synapse.storage.appservice import ApplicationService import logging @@ -88,8 +89,26 @@ class ApplicationServicesHandler(BaseHandler): # Do we know this user exists? If not, poke the user query API for # all services which match that user regex. + unknown_user = False # TODO check + if unknown_user: + user_query_services = self.get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_USERS + ) + for user_service in user_query_services: + pass # TODO poke User Query API # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. + unknown_room_alias = False # TODO check + if unknown_room_alias: + alias_query_services = self.get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_ALIASES + ) + for alias_service in alias_query_services: + pass # TODO poke Room Alias Query API # Fork off pushes to these services - XXX First cut, best effort + for service in services: + pass # TODO push event to service diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 277741fced..cdf26ee434 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -33,6 +33,9 @@ class ApplicationService(object): NS_USERS = "users" NS_ALIASES = "aliases" NS_ROOMS = "rooms" + # The ordering here is important as it is used to map database values (which + # are stored as ints representing the position in this list) to namespace + # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None): @@ -103,13 +106,21 @@ class ApplicationService(object): """ if aliases_for_event is None: aliases_for_event = [] - if restrict_to not in ApplicationService.NS_LIST: - # this is a programming error, so raise a general exception + if restrict_to and restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so fail early and raise a general + # exception raise Exception("Unexpected restrict_to value: %s". restrict_to) - return (self._matches_user(event) - or self._matches_aliases(event, aliases_for_event) - or self._matches_room_id(event)) + if not restrict_to: + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) + elif restrict_to == ApplicationService.NS_ALIASES: + return self._matches_aliases(event, aliases_for_event) + elif restrict_to == ApplicationService.NS_ROOMS: + return self._matches_room_id(event) + elif restrict_to == ApplicationService.NS_USERS: + return self._matches_user(event) def __str__(self): return "ApplicationService: %s" % (self.__dict__,) -- cgit 1.4.1 From 77a076bd25fc355c49251bcf489c0511c0f0f0af Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 13:35:17 +0000 Subject: Set combinations is | and not + --- synapse/handlers/federation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b9b2e25d1f..653ab0dbfa 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -114,9 +114,9 @@ class FederationHandler(BaseHandler): event_ids = set() if state: - event_ids += {e.event_id for e in state} + event_ids |= {e.event_id for e in state} if auth_chain: - event_ids += {e.event_id for e in auth_chain} + event_ids |= {e.event_id for e in auth_chain} seen_ids = (yield self.store.have_events(event_ids)).keys() -- cgit 1.4.1 From 6efd4d1649a539ba4bf1c884ebb90a48c2d1c8df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 13:57:54 +0000 Subject: Don't completely die if get auth_chain or querying auth_chain requests fail --- synapse/handlers/federation.py | 135 ++++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 63 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 653ab0dbfa..6727155c38 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -787,41 +787,45 @@ class FederationHandler(BaseHandler): if missing_auth: logger.debug("Missing auth: %s", missing_auth) # If we don't have all the auth events, we need to get them. - remote_auth_chain = yield self.replication_layer.get_event_auth( - origin, event.room_id, event.event_id - ) + try: + remote_auth_chain = yield self.replication_layer.get_event_auth( + origin, event.room_id, event.event_id + ) - seen_remotes = yield self.store.have_events( - [e.event_id for e in remote_auth_chain] - ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in remote_auth_chain] + ) - for e in remote_auth_chain: - if e.event_id in seen_remotes.keys(): - continue + for e in remote_auth_chain: + if e.event_id in seen_remotes.keys(): + continue - if e.event_id == event.event_id: - continue + if e.event_id == event.event_id: + continue - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in remote_auth_chain - if e.event_id in auth_ids - } - e.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in remote_auth_chain + if e.event_id in auth_ids + } + e.internal_metadata.outlier = True - logger.debug( - "do_auth %s missing_auth: %s", - event.event_id, e.event_id - ) - yield self._handle_new_event( - origin, e, auth_events=auth - ) + logger.debug( + "do_auth %s missing_auth: %s", + event.event_id, e.event_id + ) + yield self._handle_new_event( + origin, e, auth_events=auth + ) - if e.event_id in event_auth_events: - auth_events[(e.type, e.state_key)] = e - except AuthError: - pass + if e.event_id in event_auth_events: + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass + except: + # FIXME: + logger.exception("Failed to get auth chain") # FIXME: Assumes we have and stored all the state for all the # prev_events @@ -836,47 +840,52 @@ class FederationHandler(BaseHandler): auth_ids = self.auth.compute_auth_events(event, context) local_auth_chain = yield self.store.get_auth_chain(auth_ids) - # 2. Get remote difference. - result = yield self.replication_layer.query_auth( - origin, - event.room_id, - event.event_id, - local_auth_chain, - ) + try: + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) - seen_remotes = yield self.store.have_events( - [e.event_id for e in result["auth_chain"]] - ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) - # 3. Process any remote auth chain events we haven't seen. - for ev in result["auth_chain"]: - if ev.event_id in seen_remotes.keys(): - continue + # 3. Process any remote auth chain events we haven't seen. + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue - if ev.event_id == event.event_id: - continue + if ev.event_id == event.event_id: + continue - try: - auth_ids = [e_id for e_id, _ in ev.auth_events] - auth = { - (e.type, e.state_key): e for e in result["auth_chain"] - if e.event_id in auth_ids - } - ev.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in ev.auth_events] + auth = { + (e.type, e.state_key): e for e in result["auth_chain"] + if e.event_id in auth_ids + } + ev.internal_metadata.outlier = True - logger.debug( - "do_auth %s different_auth: %s", - event.event_id, e.event_id - ) + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) - yield self._handle_new_event( - origin, ev, auth_events=auth - ) + yield self._handle_new_event( + origin, ev, auth_events=auth + ) + + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev + except AuthError: + pass - if ev.event_id in event_auth_events: - auth_events[(ev.type, ev.state_key)] = ev - except AuthError: - pass + except: + # FIXME: + logger.exception("Failed to query auth chain") # 4. Look at rejects and their proofs. # TODO. -- cgit 1.4.1 From 94a5db9f4d400a345c5d8b9f7bacb0c9ccf99959 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 14:44:16 +0000 Subject: Add appservice package and move ApplicationService into it. --- synapse/appservice/__init__.py | 119 +++++++++++++++++++++++++++++++++++++++++ synapse/appservice/api.py | 15 ++++++ synapse/handlers/appservice.py | 4 +- synapse/storage/appservice.py | 105 +----------------------------------- 4 files changed, 138 insertions(+), 105 deletions(-) create mode 100644 synapse/appservice/__init__.py create mode 100644 synapse/appservice/api.py (limited to 'synapse') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py new file mode 100644 index 0000000000..f801fb5324 --- /dev/null +++ b/synapse/appservice/__init__.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.api.constants import EventTypes + +import re + + +class ApplicationService(object): + """Defines an application service. This definition is mostly what is + provided to the /register AS API. + + Provides methods to check if this service is "interested" in events. + """ + NS_USERS = "users" + NS_ALIASES = "aliases" + NS_ROOMS = "rooms" + # The ordering here is important as it is used to map database values (which + # are stored as ints representing the position in this list) to namespace + # values. + NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] + + def __init__(self, token, url=None, namespaces=None): + self.token = token + self.url = url + self.namespaces = self._check_namespaces(namespaces) + + def _check_namespaces(self, namespaces): + # Sanity check that it is of the form: + # { + # users: ["regex",...], + # aliases: ["regex",...], + # rooms: ["regex",...], + # } + if not namespaces: + return None + + for ns in ApplicationService.NS_LIST: + if type(namespaces[ns]) != list: + raise ValueError("Bad namespace value for '%s'", ns) + for regex in namespaces[ns]: + if not isinstance(regex, basestring): + raise ValueError("Expected string regex for ns '%s'", ns) + return namespaces + + def _matches_regex(self, test_string, namespace_key): + for regex in self.namespaces[namespace_key]: + if re.match(regex, test_string): + return True + return False + + def _matches_user(self, event): + if (hasattr(event, "user_id") and + self._matches_regex( + event.user_id, ApplicationService.NS_USERS)): + return True + # also check m.room.member state key + if (hasattr(event, "type") and event.type == EventTypes.Member + and hasattr(event, "state_key") + and self._matches_regex( + event.state_key, ApplicationService.NS_USERS)): + return True + return False + + def _matches_room_id(self, event): + if hasattr(event, "room_id"): + return self._matches_regex( + event.room_id, ApplicationService.NS_ROOMS + ) + return False + + def _matches_aliases(self, event, alias_list): + for alias in alias_list: + if self._matches_regex(alias, ApplicationService.NS_ALIASES): + return True + return False + + def is_interested(self, event, restrict_to=None, aliases_for_event=None): + """Check if this service is interested in this event. + + Args: + event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. + aliases_for_event(list): A list of all the known room aliases for + this event. + Returns: + bool: True if this service would like to know about this event. + """ + if aliases_for_event is None: + aliases_for_event = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so fail early and raise a general + # exception + raise Exception("Unexpected restrict_to value: %s". restrict_to) + + if not restrict_to: + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) + elif restrict_to == ApplicationService.NS_ALIASES: + return self._matches_aliases(event, aliases_for_event) + elif restrict_to == ApplicationService.NS_ROOMS: + return self._matches_room_id(event) + elif restrict_to == ApplicationService.NS_USERS: + return self._matches_user(event) + + def __str__(self): + return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py new file mode 100644 index 0000000000..803f97ea4f --- /dev/null +++ b/synapse/appservice/api.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index dac63e2245..f05b57bcb9 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError -from synapse.storage.appservice import ApplicationService +from synapse.appservice import ApplicationService import logging @@ -96,6 +96,7 @@ class ApplicationServicesHandler(BaseHandler): restrict_to=ApplicationService.NS_USERS ) for user_service in user_query_services: + # this needs to block XXX: Need to feed response back to caller pass # TODO poke User Query API # Do we know this room alias exists? If not, poke the room alias query @@ -107,6 +108,7 @@ class ApplicationServicesHandler(BaseHandler): restrict_to=ApplicationService.NS_ALIASES ) for alias_service in alias_query_services: + # this needs to block XXX: Need to feed response back to caller pass # TODO poke Room Alias Query API # Fork off pushes to these services - XXX First cut, best effort diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index cdf26ee434..48bc7e0fe6 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,119 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import re from twisted.internet import defer -from synapse.api.constants import EventTypes from synapse.api.errors import StoreError +from synapse.appservice import ApplicationService from ._base import SQLBaseStore logger = logging.getLogger(__name__) -# XXX: This feels like it should belong in a "models" module, not storage. -class ApplicationService(object): - """Defines an application service. - - Provides methods to check if this service is "interested" in events. - """ - NS_USERS = "users" - NS_ALIASES = "aliases" - NS_ROOMS = "rooms" - # The ordering here is important as it is used to map database values (which - # are stored as ints representing the position in this list) to namespace - # values. - NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - - def __init__(self, token, url=None, namespaces=None): - self.token = token - self.url = url - self.namespaces = self._check_namespaces(namespaces) - - def _check_namespaces(self, namespaces): - # Sanity check that it is of the form: - # { - # users: ["regex",...], - # aliases: ["regex",...], - # rooms: ["regex",...], - # } - if not namespaces: - return None - - for ns in ApplicationService.NS_LIST: - if type(namespaces[ns]) != list: - raise ValueError("Bad namespace value for '%s'", ns) - for regex in namespaces[ns]: - if not isinstance(regex, basestring): - raise ValueError("Expected string regex for ns '%s'", ns) - return namespaces - - def _matches_regex(self, test_string, namespace_key): - for regex in self.namespaces[namespace_key]: - if re.match(regex, test_string): - return True - return False - - def _matches_user(self, event): - if (hasattr(event, "user_id") and - self._matches_regex( - event.user_id, ApplicationService.NS_USERS)): - return True - # also check m.room.member state key - if (hasattr(event, "type") and event.type == EventTypes.Member - and hasattr(event, "state_key") - and self._matches_regex( - event.state_key, ApplicationService.NS_USERS)): - return True - return False - - def _matches_room_id(self, event): - if hasattr(event, "room_id"): - return self._matches_regex( - event.room_id, ApplicationService.NS_ROOMS - ) - return False - - def _matches_aliases(self, event, alias_list): - for alias in alias_list: - if self._matches_regex(alias, ApplicationService.NS_ALIASES): - return True - return False - - def is_interested(self, event, restrict_to=None, aliases_for_event=None): - """Check if this service is interested in this event. - - Args: - event(Event): The event to check. - restrict_to(str): The namespace to restrict regex tests to. - aliases_for_event(list): A list of all the known room aliases for - this event. - Returns: - bool: True if this service would like to know about this event. - """ - if aliases_for_event is None: - aliases_for_event = [] - if restrict_to and restrict_to not in ApplicationService.NS_LIST: - # this is a programming error, so fail early and raise a general - # exception - raise Exception("Unexpected restrict_to value: %s". restrict_to) - - if not restrict_to: - return (self._matches_user(event) - or self._matches_aliases(event, aliases_for_event) - or self._matches_room_id(event)) - elif restrict_to == ApplicationService.NS_ALIASES: - return self._matches_aliases(event, aliases_for_event) - elif restrict_to == ApplicationService.NS_ROOMS: - return self._matches_room_id(event) - elif restrict_to == ApplicationService.NS_USERS: - return self._matches_user(event) - - def __str__(self): - return "ApplicationService: %s" % (self.__dict__,) - - class ApplicationServiceCache(object): """Caches ApplicationServices and provides utility functions on top. -- cgit 1.4.1 From 0dd3aea319c13e66eb1d75b5b8a196032ee332b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 14:58:30 +0000 Subject: Keep around the old (buggy) version of the prune_event function so that we can use it to check signatures for events on old servers --- synapse/api/auth.py | 2 - synapse/events/utils.py | 79 +++++++++++++++++++++++++++ synapse/federation/federation_client.py | 96 +-------------------------------- synapse/federation/federation_server.py | 52 ++++-------------- 4 files changed, 92 insertions(+), 137 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 37e31d2b6f..3471afd7e7 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -102,8 +102,6 @@ class Auth(object): def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) - logger.debug("Got curr_state %s", curr_state) - for event in curr_state: if event.type == EventTypes.Member: try: diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 1aa952150e..65a9f70982 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -94,6 +94,85 @@ def prune_event(event): ) +def old_prune_event(event): + """This is an old and buggy version of the prune event function. The + difference between this and the new version is that when including dicts + in the content they were included as frozen_dicts rather than dicts. This + caused the JSON encoder to encode as a list of the keys rather than the + dict. + """ + event_type = event.type + + allowed_keys = [ + "event_id", + "sender", + "room_id", + "hashes", + "signatures", + "content", + "type", + "state_key", + "depth", + "prev_events", + "prev_state", + "auth_events", + "origin", + "origin_server_ts", + "membership", + ] + + event_dict = event.get_dict() + + new_content = {} + + def add_fields(*fields): + for field in fields: + if field in event.content: + # This is the line that is buggy: event.content may return + # a frozen_dict which the json encoders encode as lists rather + # than dicts. + new_content[field] = event.content[field] + + if event_type == EventTypes.Member: + add_fields("membership") + elif event_type == EventTypes.Create: + add_fields("creator") + elif event_type == EventTypes.JoinRules: + add_fields("join_rule") + elif event_type == EventTypes.PowerLevels: + add_fields( + "users", + "users_default", + "events", + "events_default", + "events_default", + "state_default", + "ban", + "kick", + "redact", + ) + elif event_type == EventTypes.Aliases: + add_fields("aliases") + + allowed_fields = { + k: v + for k, v in event_dict.items() + if k in allowed_keys + } + + allowed_fields["content"] = new_content + + allowed_fields["unsigned"] = {} + + if "age_ts" in event.unsigned: + allowed_fields["unsigned"]["age_ts"] = event.unsigned["age_ts"] + + return type(event)( + allowed_fields, + internal_metadata_dict=event.internal_metadata.get_dict() + ) + + def format_event_raw(d): return d diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 9ceb66e6f4..5fac629709 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -16,17 +16,11 @@ from twisted.internet import defer +from .federation_base import FederationBase from .units import Edu from synapse.util.logutils import log_function from synapse.events import FrozenEvent -from synapse.events.utils import prune_event - -from syutil.jsonutil import encode_canonical_json - -from synapse.crypto.event_signing import check_event_content_hash - -from synapse.api.errors import SynapseError import logging @@ -34,7 +28,7 @@ import logging logger = logging.getLogger(__name__) -class FederationClient(object): +class FederationClient(FederationBase): @log_function def send_pdu(self, pdu, destinations): """Informs the replication layer about a new PDU generated within the @@ -376,89 +370,3 @@ class FederationClient(object): event.internal_metadata.outlier = outlier return event - - @defer.inlineCallbacks - def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False): - """Takes a list of PDUs and checks the signatures and hashs of each - one. If a PDU fails its signature check then we check if we have it in - the database and if not then request if from the originating server of - that PDU. - - If a PDU fails its content hash check then it is redacted. - - The given list of PDUs are not modified, instead the function returns - a new list. - - Args: - pdu (list) - outlier (bool) - - Returns: - Deferred : A list of PDUs that have valid signatures and hashes. - """ - signed_pdus = [] - for pdu in pdus: - try: - new_pdu = yield self._check_sigs_and_hash(pdu) - signed_pdus.append(new_pdu) - except SynapseError: - # FIXME: We should handle signature failures more gracefully. - - # Check local db. - new_pdu = yield self.store.get_event( - pdu.event_id, - allow_rejected=True - ) - if new_pdu: - signed_pdus.append(new_pdu) - continue - - # Check pdu.origin - if pdu.origin != origin: - new_pdu = yield self.get_pdu( - destinations=[pdu.origin], - event_id=pdu.event_id, - outlier=outlier, - ) - - if new_pdu: - signed_pdus.append(new_pdu) - continue - - logger.warn("Failed to find copy of %s with valid signature") - - defer.returnValue(signed_pdus) - - @defer.inlineCallbacks - def _check_sigs_and_hash(self, pdu): - """Throws a SynapseError if the PDU does not have the correct - signatures. - - Returns: - FrozenEvent: Either the given event or it redacted if it failed the - content hash check. - """ - # Check signatures are correct. - redacted_event = prune_event(pdu) - redacted_pdu_json = redacted_event.get_pdu_json() - - try: - yield self.keyring.verify_json_for_server( - pdu.origin, redacted_pdu_json - ) - except SynapseError: - logger.warn( - "Signature check failed for %s redacted to %s", - encode_canonical_json(pdu.get_pdu_json()), - encode_canonical_json(redacted_pdu_json), - ) - raise - - if not check_event_content_hash(pdu): - logger.warn( - "Event content has been tampered, redacting %s, %s", - pdu.event_id, encode_canonical_json(pdu.get_dict()) - ) - defer.returnValue(redacted_event) - - defer.returnValue(pdu) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 5fbd8b19de..97dca3f84c 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -16,6 +16,7 @@ from twisted.internet import defer +from .federation_base import FederationBase from .units import Transaction, Edu from synapse.util.logutils import log_function @@ -35,7 +36,7 @@ import logging logger = logging.getLogger(__name__) -class FederationServer(object): +class FederationServer(FederationBase): def set_handler(self, handler): """Sets the handler that the replication layer will use to communicate receipt of new PDUs from other home servers. The required methods are @@ -251,17 +252,20 @@ class FederationServer(object): Deferred: Results in `dict` with the same format as `content` """ auth_chain = [ - (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + self.event_from_pdu_json(e) for e in content["auth_chain"] ] - missing = [ - (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) - for e in content.get("missing", []) - ] + signed_auth = yield self._check_sigs_and_hash_and_fetch( + origin, auth_chain, outlier=True + ) ret = yield self.handler.on_query_auth( - origin, event_id, auth_chain, content.get("rejects", []), missing + origin, + event_id, + signed_auth, + content.get("rejects", []), + content.get("missing", []), ) time_now = self._clock.time_msec() @@ -426,37 +430,3 @@ class FederationServer(object): event.internal_metadata.outlier = outlier return event - - @defer.inlineCallbacks - def _check_sigs_and_hash(self, pdu): - """Throws a SynapseError if the PDU does not have the correct - signatures. - - Returns: - FrozenEvent: Either the given event or it redacted if it failed the - content hash check. - """ - # Check signatures are correct. - redacted_event = prune_event(pdu) - redacted_pdu_json = redacted_event.get_pdu_json() - - try: - yield self.keyring.verify_json_for_server( - pdu.origin, redacted_pdu_json - ) - except SynapseError: - logger.warn( - "Signature check failed for %s redacted to %s", - encode_canonical_json(pdu.get_pdu_json()), - encode_canonical_json(redacted_pdu_json), - ) - raise - - if not check_event_content_hash(pdu): - logger.warn( - "Event content has been tampered, redacting %s, %s", - pdu.event_id, encode_canonical_json(pdu.get_dict()) - ) - defer.returnValue(redacted_event) - - defer.returnValue(pdu) -- cgit 1.4.1 From 7b810e136ef2040fe55a778d4a4a790cdbd0f84c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 15:00:42 +0000 Subject: Add new FederationBase --- synapse/federation/federation_base.py | 126 ++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 synapse/federation/federation_base.py (limited to 'synapse') diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py new file mode 100644 index 0000000000..d26a2396ac --- /dev/null +++ b/synapse/federation/federation_base.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from synapse.events.utils import prune_event, old_prune_event + +from syutil.jsonutil import encode_canonical_json + +from synapse.crypto.event_signing import check_event_content_hash + +from synapse.api.errors import SynapseError + +import logging + + +logger = logging.getLogger(__name__) + + +class FederationBase(object): + @defer.inlineCallbacks + def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False): + """Takes a list of PDUs and checks the signatures and hashs of each + one. If a PDU fails its signature check then we check if we have it in + the database and if not then request if from the originating server of + that PDU. + + If a PDU fails its content hash check then it is redacted. + + The given list of PDUs are not modified, instead the function returns + a new list. + + Args: + pdu (list) + outlier (bool) + + Returns: + Deferred : A list of PDUs that have valid signatures and hashes. + """ + signed_pdus = [] + for pdu in pdus: + try: + new_pdu = yield self._check_sigs_and_hash(pdu) + signed_pdus.append(new_pdu) + except SynapseError: + # FIXME: We should handle signature failures more gracefully. + + # Check local db. + new_pdu = yield self.store.get_event( + pdu.event_id, + allow_rejected=True + ) + if new_pdu: + signed_pdus.append(new_pdu) + continue + + # Check pdu.origin + if pdu.origin != origin: + new_pdu = yield self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + ) + + if new_pdu: + signed_pdus.append(new_pdu) + continue + + logger.warn("Failed to find copy of %s with valid signature") + + defer.returnValue(signed_pdus) + + @defer.inlineCallbacks + def _check_sigs_and_hash(self, pdu): + """Throws a SynapseError if the PDU does not have the correct + signatures. + + Returns: + FrozenEvent: Either the given event or it redacted if it failed the + content hash check. + """ + # Check signatures are correct. + redacted_event = prune_event(pdu) + redacted_pdu_json = redacted_event.get_pdu_json() + + old_redacted = old_prune_event(pdu) + old_redacted_pdu_json = old_redacted.get_pdu_json() + + try: + try: + yield self.keyring.verify_json_for_server( + pdu.origin, old_redacted_pdu_json + ) + except SynapseError: + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) + except SynapseError: + logger.warn( + "Signature check failed for %s redacted to %s", + encode_canonical_json(pdu.get_pdu_json()), + encode_canonical_json(redacted_pdu_json), + ) + raise + + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s, %s", + pdu.event_id, encode_canonical_json(pdu.get_dict()) + ) + defer.returnValue(redacted_event) + + defer.returnValue(pdu) \ No newline at end of file -- cgit 1.4.1 From 8dae5c81085458a3a0b3dc92278e8c317d5de204 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 15:01:12 +0000 Subject: Remove unused imports --- synapse/federation/federation_server.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 97dca3f84c..4742ca9390 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -22,11 +22,6 @@ from .units import Transaction, Edu from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext from synapse.events import FrozenEvent -from synapse.events.utils import prune_event - -from syutil.jsonutil import encode_canonical_json - -from synapse.crypto.event_signing import check_event_content_hash from synapse.api.errors import FederationError, SynapseError -- cgit 1.4.1 From 9bace3a36751deed141225ccabd5bebecebc25f3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 15:32:17 +0000 Subject: Actually, the old prune_event function was non-deterministic, so no point keeping it around :( --- synapse/events/utils.py | 79 ----------------------------------- synapse/federation/federation_base.py | 16 ++----- 2 files changed, 4 insertions(+), 91 deletions(-) (limited to 'synapse') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 65a9f70982..1aa952150e 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -94,85 +94,6 @@ def prune_event(event): ) -def old_prune_event(event): - """This is an old and buggy version of the prune event function. The - difference between this and the new version is that when including dicts - in the content they were included as frozen_dicts rather than dicts. This - caused the JSON encoder to encode as a list of the keys rather than the - dict. - """ - event_type = event.type - - allowed_keys = [ - "event_id", - "sender", - "room_id", - "hashes", - "signatures", - "content", - "type", - "state_key", - "depth", - "prev_events", - "prev_state", - "auth_events", - "origin", - "origin_server_ts", - "membership", - ] - - event_dict = event.get_dict() - - new_content = {} - - def add_fields(*fields): - for field in fields: - if field in event.content: - # This is the line that is buggy: event.content may return - # a frozen_dict which the json encoders encode as lists rather - # than dicts. - new_content[field] = event.content[field] - - if event_type == EventTypes.Member: - add_fields("membership") - elif event_type == EventTypes.Create: - add_fields("creator") - elif event_type == EventTypes.JoinRules: - add_fields("join_rule") - elif event_type == EventTypes.PowerLevels: - add_fields( - "users", - "users_default", - "events", - "events_default", - "events_default", - "state_default", - "ban", - "kick", - "redact", - ) - elif event_type == EventTypes.Aliases: - add_fields("aliases") - - allowed_fields = { - k: v - for k, v in event_dict.items() - if k in allowed_keys - } - - allowed_fields["content"] = new_content - - allowed_fields["unsigned"] = {} - - if "age_ts" in event.unsigned: - allowed_fields["unsigned"]["age_ts"] = event.unsigned["age_ts"] - - return type(event)( - allowed_fields, - internal_metadata_dict=event.internal_metadata.get_dict() - ) - - def format_event_raw(d): return d diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index d26a2396ac..27c5918e0f 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -16,7 +16,7 @@ from twisted.internet import defer -from synapse.events.utils import prune_event, old_prune_event +from synapse.events.utils import prune_event from syutil.jsonutil import encode_canonical_json @@ -96,18 +96,10 @@ class FederationBase(object): redacted_event = prune_event(pdu) redacted_pdu_json = redacted_event.get_pdu_json() - old_redacted = old_prune_event(pdu) - old_redacted_pdu_json = old_redacted.get_pdu_json() - try: - try: - yield self.keyring.verify_json_for_server( - pdu.origin, old_redacted_pdu_json - ) - except SynapseError: - yield self.keyring.verify_json_for_server( - pdu.origin, redacted_pdu_json - ) + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) except SynapseError: logger.warn( "Signature check failed for %s redacted to %s", -- cgit 1.4.1 From 9a71add1c077130bf2cf3998ab0dd2226ba0c75b Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 3 Feb 2015 16:06:31 +0000 Subject: Use set_tweak instead of set_sound --- synapse/push/__init__.py | 4 ++-- synapse/push/baserules.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 28e5dae81d..00f3513c23 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -400,8 +400,8 @@ def _tweaks_for_actions(actions): for a in actions: if not isinstance(a, dict): continue - if 'set_sound' in a: - tweaks['sound'] = a['set_sound'] + if 'set_tweak' in a and 'value' in a: + tweaks[a['set_tweak']] = a['value'] return tweaks diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 382de118e0..376d1d4d33 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -38,7 +38,8 @@ def make_base_rules(user_name): 'actions': [ 'notify', { - 'set_sound': 'default' + 'set_tweak': 'sound', + 'value': 'default' } ] } -- cgit 1.4.1 From 7dd1c5c542d58464085b11ababe746c6a60515ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 16:12:04 +0000 Subject: Neaten the handling of state and auth_chain up a bit --- synapse/handlers/federation.py | 57 ++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 27 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6727155c38..86953bf8c8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -30,6 +30,7 @@ from synapse.types import UserID from twisted.internet import defer +import itertools import logging @@ -112,14 +113,6 @@ class FederationHandler(BaseHandler): logger.debug("Event: %s", event) - event_ids = set() - if state: - event_ids |= {e.event_id for e in state} - if auth_chain: - event_ids |= {e.event_id for e in auth_chain} - - seen_ids = (yield self.store.have_events(event_ids)).keys() - # FIXME (erikj): Awful hack to make the case where we are not currently # in the room work current_state = None @@ -131,27 +124,37 @@ class FederationHandler(BaseHandler): logger.debug("Got event for room we're not in.") current_state = state + event_ids = set() + if state: + event_ids |= {e.event_id for e in state} + if auth_chain: + event_ids |= {e.event_id for e in auth_chain} + + seen_ids = (yield self.store.have_events(event_ids)).keys() + if state and auth_chain is not None: - for list_of_pdus in [auth_chain, state]: - for e in list_of_pdus: - if e.event_id in seen_ids: - continue + # If we have any state or auth_chain given to us by the replication + # layer, then we should handle them (if we haven't before.) + for e in itertools.chain(auth_chain, state): + if e.event_id in seen_ids: + continue - e.internal_metadata.outlier = True - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - yield self._handle_new_event( - origin, e, auth_events=auth - ) - except: - logger.exception( - "Failed to handle state event %s", - e.event_id, - ) + e.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + seen_ids.add(e.event_id) + except: + logger.exception( + "Failed to handle state event %s", + e.event_id, + ) try: yield self._handle_new_event( -- cgit 1.4.1 From 3c39f42a0526186f85e69988504fff6adbf41d91 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 16:14:19 +0000 Subject: New line --- synapse/federation/federation_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 27c5918e0f..a990aec4fd 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -115,4 +115,4 @@ class FederationBase(object): ) defer.returnValue(redacted_event) - defer.returnValue(pdu) \ No newline at end of file + defer.returnValue(pdu) -- cgit 1.4.1 From dc7bb70f22edf8ef0631c961f2c77a82de7c76d5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 3 Feb 2015 16:51:07 +0000 Subject: s/instance_handle/profile_tag/ --- synapse/push/__init__.py | 8 ++++---- synapse/push/httppusher.py | 4 ++-- synapse/push/pusherpool.py | 12 ++++++------ synapse/rest/client/v1/push_rule.py | 28 ++++++++++++++-------------- synapse/rest/client/v1/pusher.py | 4 ++-- synapse/storage/pusher.py | 14 +++++++------- synapse/storage/schema/delta/v12.sql | 2 +- synapse/storage/schema/pusher.sql | 2 +- 8 files changed, 37 insertions(+), 37 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 00f3513c23..8c6f0a6571 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -37,14 +37,14 @@ class Pusher(object): INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$") - def __init__(self, _hs, instance_handle, user_name, app_id, + def __init__(self, _hs, profile_tag, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() - self.instance_handle = instance_handle + self.profile_tag = profile_tag self.user_name = user_name self.app_id = app_id self.app_display_name = app_display_name @@ -147,9 +147,9 @@ class Pusher(object): return False return fnmatch.fnmatch(val.upper(), pat.upper()) elif condition['kind'] == 'device': - if 'instance_handle' not in condition: + if 'profile_tag' not in condition: return True - return condition['instance_handle'] == self.instance_handle + return condition['profile_tag'] == self.profile_tag elif condition['kind'] == 'contains_display_name': # This is special because display names can be different # between rooms and so you can't really hard code it in a rule. diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 7c6953c989..5788db4eba 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -24,12 +24,12 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, instance_handle, user_name, app_id, + def __init__(self, _hs, profile_tag, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, - instance_handle, + profile_tag, user_name, app_id, app_display_name, diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 4892c21e7b..5a525befd7 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -55,7 +55,7 @@ class PusherPool: self._start_pushers(pushers) @defer.inlineCallbacks - def add_pusher(self, user_name, instance_handle, kind, app_id, + def add_pusher(self, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, @@ -64,7 +64,7 @@ class PusherPool: self._create_pusher({ "user_name": user_name, "kind": kind, - "instance_handle": instance_handle, + "profile_tag": profile_tag, "app_id": app_id, "app_display_name": app_display_name, "device_display_name": device_display_name, @@ -77,18 +77,18 @@ class PusherPool: "failing_since": None }) yield self._add_pusher_to_store( - user_name, instance_handle, kind, app_id, + user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data ) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, instance_handle, kind, app_id, + def _add_pusher_to_store(self, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): yield self.store.add_pusher( user_name=user_name, - instance_handle=instance_handle, + profile_tag=profile_tag, kind=kind, app_id=app_id, app_display_name=app_display_name, @@ -104,7 +104,7 @@ class PusherPool: if pusherdict['kind'] == 'http': return HttpPusher( self.hs, - instance_handle=pusherdict['instance_handle'], + profile_tag=pusherdict['profile_tag'], user_name=pusherdict['user_name'], app_id=pusherdict['app_id'], app_display_name=pusherdict['app_display_name'], diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index faa7919fbb..348adb9c0d 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -112,7 +112,7 @@ class PushRuleRestServlet(ClientV1RestServlet): if device: conditions.append({ 'kind': 'device', - 'instance_handle': device + 'profile_tag': device }) if 'actions' not in req_obj: @@ -195,7 +195,7 @@ class PushRuleRestServlet(ClientV1RestServlet): for r in rules: conditions = json.loads(r['conditions']) - ih = _instance_handle_from_conditions(conditions) + ih = _profile_tag_from_conditions(conditions) if ih == spec['device'] and r['priority_class'] == priority_class: yield self.hs.get_datastore().delete_push_rule( user.to_string(), spec['rule_id'] @@ -239,19 +239,19 @@ class PushRuleRestServlet(ClientV1RestServlet): if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device rule - instance_handle = _instance_handle_from_conditions(r["conditions"]) + profile_tag = _profile_tag_from_conditions(r["conditions"]) r = _strip_device_condition(r) - if not instance_handle: + if not profile_tag: continue - if instance_handle not in rules['device']: - rules['device'][instance_handle] = {} - rules['device'][instance_handle] = ( + if profile_tag not in rules['device']: + rules['device'][profile_tag] = {} + rules['device'][profile_tag] = ( _add_empty_priority_class_arrays( - rules['device'][instance_handle] + rules['device'][profile_tag] ) ) - rulearray = rules['device'][instance_handle][template_name] + rulearray = rules['device'][profile_tag][template_name] else: rulearray = rules['global'][template_name] @@ -282,13 +282,13 @@ class PushRuleRestServlet(ClientV1RestServlet): if path[0] == '': defer.returnValue((200, rules['device'])) - instance_handle = path[0] + profile_tag = path[0] path = path[1:] - if instance_handle not in rules['device']: + if profile_tag not in rules['device']: ret = {} ret = _add_empty_priority_class_arrays(ret) defer.returnValue((200, ret)) - ruleset = rules['device'][instance_handle] + ruleset = rules['device'][profile_tag] result = _filter_ruleset_with_path(ruleset, path) defer.returnValue((200, result)) else: @@ -304,14 +304,14 @@ def _add_empty_priority_class_arrays(d): return d -def _instance_handle_from_conditions(conditions): +def _profile_tag_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the device rule if there is one """ for c in conditions: if c['kind'] == 'device': - return c['instance_handle'] + return c['profile_tag'] return None diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 353a4a6589..e10d2576d2 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -41,7 +41,7 @@ class PusherRestServlet(ClientV1RestServlet): ) defer.returnValue((200, {})) - reqd = ['instance_handle', 'kind', 'app_id', 'app_display_name', + reqd = ['profile_tag', 'kind', 'app_id', 'app_display_name', 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] for i in reqd: @@ -54,7 +54,7 @@ class PusherRestServlet(ClientV1RestServlet): try: yield pusher_pool.add_pusher( user_name=user.to_string(), - instance_handle=content['instance_handle'], + profile_tag=content['profile_tag'], kind=content['kind'], app_id=content['app_id'], app_display_name=content['app_display_name'], diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index f253c9e2c3..e2a662a6c7 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -29,7 +29,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( - "SELECT id, user_name, kind, instance_handle, app_id," + "SELECT id, user_name, kind, profile_tag, app_id," "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers " @@ -45,7 +45,7 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "instance_handle": r[3], + "profile_tag": r[3], "app_id": r[4], "app_display_name": r[5], "device_display_name": r[6], @@ -64,7 +64,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers(self): sql = ( - "SELECT id, user_name, kind, instance_handle, app_id," + "SELECT id, user_name, kind, profile_tag, app_id," "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers" @@ -77,7 +77,7 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "instance_handle": r[3], + "profile_tag": r[3], "app_id": r[4], "app_display_name": r[5], "device_display_name": r[6], @@ -94,7 +94,7 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, instance_handle, kind, app_id, + def add_pusher(self, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: @@ -107,7 +107,7 @@ class PusherStore(SQLBaseStore): dict( user_name=user_name, kind=kind, - instance_handle=instance_handle, + profile_tag=profile_tag, app_display_name=app_display_name, device_display_name=device_display_name, ts=pushkey_ts, @@ -158,7 +158,7 @@ class PushersTable(Table): "id", "user_name", "kind", - "instance_handle", + "profile_tag", "app_id", "app_display_name", "device_display_name", diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql index a6867cba62..16c2258ca4 100644 --- a/synapse/storage/schema/delta/v12.sql +++ b/synapse/storage/schema/delta/v12.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS rejections( CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, - instance_handle varchar(32) NOT NULL, + profile_tag varchar(32) NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 8c4dfd5c1b..3735b11547 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, - instance_handle varchar(32) NOT NULL, + profile_tag varchar(32) NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, -- cgit 1.4.1 From 02be8da5e11d9abcfc962f962bbc4e9940b69199 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 17:34:07 +0000 Subject: Add doc to get_event --- synapse/storage/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 93aefe0c48..93ab26fcd1 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -131,6 +131,21 @@ class DataStore(RoomMemberStore, RoomStore, def get_event(self, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False, allow_none=False): + """Get an event from the database by event_id. + + Args: + event_id (str): The event_id of the event to fetch + check_redacted (bool): If True, check if event has been redacted + and redact it. + get_prev_content (bool): If True and event is a state event, + include the previous states content in the unsigned field. + allow_rejected (bool): If True return rejected events. + allow_none (bool): If True, return None if no event found, if + False throw an exception. + + Returns: + Deferred : A FrozenEvent. + """ event = yield self.runInteraction( "get_event", self._get_event_txn, event_id, -- cgit 1.4.1 From c0462dbf1533f285f632dcb0a74c0ef0c3e2475b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 10:16:51 +0000 Subject: Rearrange persist_event so that do all the queries that need to be done before returning early if we have already persisted that event. --- synapse/events/__init__.py | 2 +- synapse/handlers/federation.py | 2 + synapse/storage/__init__.py | 145 +++++++++++++++++++++-------------------- 3 files changed, 77 insertions(+), 72 deletions(-) (limited to 'synapse') diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index bf07951027..8f0c6e959f 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -77,7 +77,7 @@ class EventBase(object): return self.content["membership"] def is_state(self): - return hasattr(self, "state_key") + return hasattr(self, "state_key") and self.state_key is not None def get_dict(self): d = dict(self._event_dict) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 86953bf8c8..0876589e31 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -515,6 +515,8 @@ class FederationHandler(BaseHandler): "Failed to get destination from event %s", s.event_id ) + destinations.remove(origin) + logger.debug( "on_send_join_request: Sending event: %s, signatures: %s", event.event_id, diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 93ab26fcd1..30ce378900 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -163,19 +163,70 @@ class DataStore(RoomMemberStore, RoomStore, def _persist_event_txn(self, txn, event, context, backfilled, stream_ordering=None, is_new_state=True, current_state=None): - if event.type == EventTypes.Member: - self._store_room_member_txn(txn, event) - elif event.type == EventTypes.Feedback: - self._store_feedback_txn(txn, event) - elif event.type == EventTypes.Name: - self._store_room_name_txn(txn, event) - elif event.type == EventTypes.Topic: - self._store_room_topic_txn(txn, event) - elif event.type == EventTypes.Redaction: - self._store_redaction(txn, event) + + # We purposefully do this first since if we include a `current_state` + # key, we *want* to update the `current_state_events` table + if current_state: + txn.execute( + "DELETE FROM current_state_events WHERE room_id = ?", + (event.room_id,) + ) + + for s in current_state: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": s.event_id, + "room_id": s.room_id, + "type": s.type, + "state_key": s.state_key, + }, + or_replace=True, + ) + + if event.is_state() and is_new_state: + if not backfilled and not context.rejected: + self._simple_insert_txn( + txn, + table="state_forward_extremities", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + or_replace=True, + ) + + for prev_state_id, _ in event.prev_state: + self._simple_delete_txn( + txn, + table="state_forward_extremities", + keyvalues={ + "event_id": prev_state_id, + } + ) outlier = event.internal_metadata.is_outlier() + if not outlier: + self._store_state_groups_txn(txn, event, context) + + self._update_min_depth_for_room_txn( + txn, + event.room_id, + event.depth + ) + + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + have_persisted = self._simple_select_one_onecol_txn( txn, table="event_json", @@ -209,6 +260,17 @@ class DataStore(RoomMemberStore, RoomStore, ) return + if event.type == EventTypes.Member: + self._store_room_member_txn(txn, event) + elif event.type == EventTypes.Feedback: + self._store_feedback_txn(txn, event) + elif event.type == EventTypes.Name: + self._store_room_name_txn(txn, event) + elif event.type == EventTypes.Topic: + self._store_room_topic_txn(txn, event) + elif event.type == EventTypes.Redaction: + self._store_redaction(txn, event) + event_dict = { k: v for k, v in event.get_dict().items() @@ -273,41 +335,10 @@ class DataStore(RoomMemberStore, RoomStore, ) raise _RollbackButIsFineException("_persist_event") - self._handle_prev_events( - txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, - ) - - if not outlier: - self._store_state_groups_txn(txn, event, context) - if context.rejected: self._store_rejections_txn(txn, event.event_id, context.rejected) - if current_state: - txn.execute( - "DELETE FROM current_state_events WHERE room_id = ?", - (event.room_id,) - ) - - for s in current_state: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": s.event_id, - "room_id": s.room_id, - "type": s.type, - "state_key": s.state_key, - }, - or_replace=True, - ) - - is_state = hasattr(event, "state_key") and event.state_key is not None - if is_state: + if event.is_state(): vals = { "event_id": event.event_id, "room_id": event.room_id, @@ -315,6 +346,7 @@ class DataStore(RoomMemberStore, RoomStore, "state_key": event.state_key, } + # TODO: How does this work with backfilling? if hasattr(event, "replaces_state"): vals["prev_state"] = event.replaces_state @@ -351,28 +383,6 @@ class DataStore(RoomMemberStore, RoomStore, or_ignore=True, ) - if not backfilled and not context.rejected: - self._simple_insert_txn( - txn, - table="state_forward_extremities", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - or_replace=True, - ) - - for prev_state_id, _ in event.prev_state: - self._simple_delete_txn( - txn, - table="state_forward_extremities", - keyvalues={ - "event_id": prev_state_id, - } - ) - for hash_alg, hash_base64 in event.hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_event_content_hash_txn( @@ -403,13 +413,6 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, ref_alg, ref_hash_bytes ) - if not outlier: - self._update_min_depth_for_room_txn( - txn, - event.room_id, - event.depth - ) - def _store_redaction(self, txn, event): txn.execute( "INSERT OR IGNORE INTO redactions " -- cgit 1.4.1 From f275ba49bbcb86e111ed0d66dd99da617220ae79 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 10:36:28 +0000 Subject: Fix state resolution to remember join_rules is a type of auth event. --- synapse/state.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 8a056ee955..6a6fb8aea0 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -37,7 +37,10 @@ def _get_state_key_from_event(event): KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) -AuthEventTypes = (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) +AuthEventTypes = ( + EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels, + EventTypes.JoinRules, +) class StateHandler(object): @@ -258,6 +261,15 @@ class StateHandler(object): auth_events.update(resolved_state) + for key, events in conflicted_state.items(): + if key[0] == EventTypes.JoinRules: + resolved_state[key] = self._resolve_auth_events( + events, + auth_events + ) + + auth_events.update(resolved_state) + for key, events in conflicted_state.items(): if key[0] == EventTypes.Member: resolved_state[key] = self._resolve_auth_events( -- cgit 1.4.1 From 03d415a6a23300e36b5e6c35080ac4dd8ab06815 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 10:40:59 +0000 Subject: Brief comment on why we do some things on every call to persist_event and not others --- synapse/storage/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 30ce378900..a63c59a8a2 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -239,6 +239,12 @@ class DataStore(RoomMemberStore, RoomStore, event.internal_metadata.get_dict() ) + # If we have already persisted this event, we don't need to do any + # more processing. + # The processing above must be done on every call to persist event, + # since they might not have happened on previous calls. For example, + # if we are persisting an event that we had persisted as an outlier, + # but is no longer one. if have_persisted: if not outlier: sql = ( -- cgit 1.4.1 From 17753f0c20d0d8190095c5a3183630b78bf9650c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 11:19:18 +0000 Subject: Add stub ApplicationServiceApi and glue it with the handler. --- synapse/appservice/__init__.py | 3 ++- synapse/appservice/api.py | 21 +++++++++++++++++++++ synapse/handlers/appservice.py | 18 +++++++++++++++--- synapse/storage/appservice.py | 1 + 4 files changed, 39 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f801fb5324..92f64619c9 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -31,10 +31,11 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - def __init__(self, token, url=None, namespaces=None): + def __init__(self, token, url=None, namespaces=None, txn_id=None): self.token = token self.url = url self.namespaces = self._check_namespaces(namespaces) + self.txn_id = None def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 803f97ea4f..158aded66e 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,3 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. + +class ApplicationServiceApi(object): + """This class manages HS -> AS communications, including querying and + pushing. + """ + + def __init__(self, hs): + self.hs_token = "_hs_token_" # TODO extract hs token + + def query_user(self, service, user_id): + pass + + def query_alias(self, service, alias): + pass + + def push_bulk(self, service, events): + pass + + def push(self, service, event): + pass + diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index f05b57bcb9..9cdeaa2d94 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService +from synapse.appservice.api import ApplicationServiceApi import logging @@ -29,6 +30,7 @@ class ApplicationServicesHandler(BaseHandler): def __init__(self, hs): super(ApplicationServicesHandler, self).__init__(hs) + self.appservice_api = ApplicationServiceApi(hs) @defer.inlineCallbacks def register(self, app_service): @@ -97,7 +99,12 @@ class ApplicationServicesHandler(BaseHandler): ) for user_service in user_query_services: # this needs to block XXX: Need to feed response back to caller - pass # TODO poke User Query API + is_known_user = self.appservice_api.query_user( + user_service, event + ) + if is_known_user: + # the user exists now,so don't query more ASes. + break # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. @@ -109,8 +116,13 @@ class ApplicationServicesHandler(BaseHandler): ) for alias_service in alias_query_services: # this needs to block XXX: Need to feed response back to caller - pass # TODO poke Room Alias Query API + is_known_alias = self.appservice_api.query_alias( + alias_service, event + ) + if is_known_alias: + # the alias exists now so don't query more ASes. + break # Fork off pushes to these services - XXX First cut, best effort for service in services: - pass # TODO push event to service + self.appservice_api.push(service, event) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 48bc7e0fe6..abb617f049 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -216,6 +216,7 @@ class ApplicationServiceStore(SQLBaseStore): except IndexError: logger.error("Bad namespace enum '%s'. %s", ns_int, res) + # TODO get last successful txn id f.e. service for service in services.values(): logger.info("Found application service: %s", service) self.cache.services.append(ApplicationService( -- cgit 1.4.1 From 525a218b2b072b24721c9c9efae42aae21388fc8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 12:24:20 +0000 Subject: Begin to add unit tests for appservice glue and regex testing. --- synapse/appservice/__init__.py | 14 ++++++-- synapse/handlers/appservice.py | 8 +++-- tests/appservice/__init__.py | 14 ++++++++ tests/appservice/test_appservice.py | 58 +++++++++++++++++++++++++++++++ tests/handlers/test_appservice.py | 68 +++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 tests/appservice/__init__.py create mode 100644 tests/appservice/test_appservice.py create mode 100644 tests/handlers/test_appservice.py (limited to 'synapse') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 92f64619c9..0c7f58574e 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -14,8 +14,11 @@ # limitations under the License. from synapse.api.constants import EventTypes +import logging import re +logger = logging.getLogger(__name__) + class ApplicationService(object): """Defines an application service. This definition is mostly what is @@ -56,15 +59,22 @@ class ApplicationService(object): return namespaces def _matches_regex(self, test_string, namespace_key): + if not isinstance(test_string, basestring): + logger.warning( + "Expected a string to test regex against, but got %s", + test_string + ) + return False + for regex in self.namespaces[namespace_key]: if re.match(regex, test_string): return True return False def _matches_user(self, event): - if (hasattr(event, "user_id") and + if (hasattr(event, "sender") and self._matches_regex( - event.user_id, ApplicationService.NS_USERS)): + event.sender, ApplicationService.NS_USERS)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 9cdeaa2d94..3188c60f3d 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -26,10 +26,14 @@ import logging logger = logging.getLogger(__name__) -class ApplicationServicesHandler(BaseHandler): +# NB: Purposefully not inheriting BaseHandler since that contains way too much +# setup code which this handler does not need or use. This makes testing a lot +# easier. +class ApplicationServicesHandler(object): def __init__(self, hs): - super(ApplicationServicesHandler, self).__init__(hs) + self.store = hs.get_datastore() + self.hs = hs self.appservice_api = ApplicationServiceApi(hs) @defer.inlineCallbacks diff --git a/tests/appservice/__init__.py b/tests/appservice/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/tests/appservice/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py new file mode 100644 index 0000000000..5cfd26daa6 --- /dev/null +++ b/tests/appservice/test_appservice.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.appservice import ApplicationService + +from mock import Mock, PropertyMock +from tests import unittest + + +class ApplicationServiceTestCase(unittest.TestCase): + + def setUp(self): + self.service = ApplicationService( + url="some_url", + token="some_token", + namespaces={ + ApplicationService.NS_USERS: [], + ApplicationService.NS_ROOMS: [], + ApplicationService.NS_ALIASES: [] + } + ) + self.event = Mock( + type="m.something", room_id="!foo:bar", sender="@someone:somewhere" + ) + + def test_regex_user_id_prefix_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_user_id_prefix_no_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.assertFalse(self.service.is_interested(self.event)) + + def test_regex_room_member_is_checked(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.event.type = "m.room.member" + self.event.state_key = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py new file mode 100644 index 0000000000..9c464e7fbc --- /dev/null +++ b/tests/handlers/test_appservice.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer +from .. import unittest + +from synapse.handlers.appservice import ApplicationServicesHandler + +from collections import namedtuple +from mock import Mock + +# TODO: Should this be a more general thing? tests/api/test_filtering.py uses it +MockEvent = namedtuple("MockEvent", "sender type room_id") + + +class AppServiceHandlerTestCase(unittest.TestCase): + """ Tests the ApplicationServicesHandler. """ + + def setUp(self): + self.mock_store = Mock() + self.mock_as_api = Mock() + hs = Mock() + hs.get_datastore = Mock(return_value=self.mock_store) + self.handler = ApplicationServicesHandler(hs) # thing being tested + + # FIXME Would be nice to DI this rather than monkey patch:( + if not hasattr(self.handler, "appservice_api"): + # someone probably updated the handler but not the tests. Fail fast. + raise Exception("Test expected handler.appservice_api to exist.") + self.handler.appservice_api = self.mock_as_api + + def test_notify_interested_services(self): + interested_service = self._mkservice(is_interested=True) + services = [ + self._mkservice(is_interested=False), + interested_service, + self._mkservice(is_interested=False) + ] + + self.mock_store.get_app_services = Mock(return_value=services) + + event = MockEvent( + sender="@someone:anywhere", + type="m.room.message", + room_id="!foo:bar" + ) + self.mock_as_api.push = Mock() + self.handler.notify_interested_services(event) + self.mock_as_api.push.assert_called_once_with(interested_service, event) + + def _mkservice(self, is_interested): + service = Mock() + service.is_interested = Mock(return_value=is_interested) + service.token = "mock_service_token" + service.url = "mock_service_url" + return service -- cgit 1.4.1 From ff78eded015b7596e883623bf826aa579662e766 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 13:55:10 +0000 Subject: Retry make_join --- synapse/federation/federation_client.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 5fac629709..d6b8c43916 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -251,16 +251,21 @@ class FederationClient(FederationBase): defer.returnValue(signed_auth) @defer.inlineCallbacks - def make_join(self, destination, room_id, user_id): - ret = yield self.transport_layer.make_join( - destination, room_id, user_id - ) + def make_join(self, destinations, room_id, user_id): + for destination in destinations: + try: + ret = yield self.transport_layer.make_join( + destination, room_id, user_id + ) - pdu_dict = ret["event"] + pdu_dict = ret["event"] - logger.debug("Got response to make_join: %s", pdu_dict) + logger.debug("Got response to make_join: %s", pdu_dict) - defer.returnValue(self.event_from_pdu_json(pdu_dict)) + defer.returnValue(self.event_from_pdu_json(pdu_dict)) + break + except Exception as e: + logger.warn("Failed to make_join via %s", destination) @defer.inlineCallbacks def send_join(self, destination, pdu): -- cgit 1.4.1 From 650e32d45580ddd13826364291bda6760c014df9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 14:06:42 +0000 Subject: Change context.auth_events to what the auth_events would be bases on context.current_state, rather than based on the auth_events from the event. --- synapse/api/auth.py | 12 ++++++------ synapse/handlers/federation.py | 4 +++- synapse/state.py | 8 ++++++-- 3 files changed, 15 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3471afd7e7..7105ee21dc 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -358,7 +358,7 @@ class Auth(object): def add_auth_events(self, builder, context): yield run_on_reactor() - auth_ids = self.compute_auth_events(builder, context) + auth_ids = self.compute_auth_events(builder, context.current_state) auth_events_entries = yield self.store.add_event_hashes( auth_ids @@ -372,26 +372,26 @@ class Auth(object): if v.event_id in auth_ids } - def compute_auth_events(self, event, context): + def compute_auth_events(self, event, current_state): if event.type == EventTypes.Create: return [] auth_ids = [] key = (EventTypes.PowerLevels, "", ) - power_level_event = context.current_state.get(key) + power_level_event = current_state.get(key) if power_level_event: auth_ids.append(power_level_event.event_id) key = (EventTypes.JoinRules, "", ) - join_rule_event = context.current_state.get(key) + join_rule_event = current_state.get(key) key = (EventTypes.Member, event.user_id, ) - member_event = context.current_state.get(key) + member_event = current_state.get(key) key = (EventTypes.Create, "", ) - create_event = context.current_state.get(key) + create_event = current_state.get(key) if create_event: auth_ids.append(create_event.event_id) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0876589e31..2e2c23ef63 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -842,7 +842,9 @@ class FederationHandler(BaseHandler): logger.debug("Different auth: %s", different_auth) # 1. Get what we think is the auth chain. - auth_ids = self.auth.compute_auth_events(event, context) + auth_ids = self.auth.compute_auth_events( + event, context.current_state + ) local_auth_chain = yield self.store.get_auth_chain(auth_ids) try: diff --git a/synapse/state.py b/synapse/state.py index 6a6fb8aea0..695a5e7ac4 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -103,7 +103,9 @@ class StateHandler(object): context.state_group = None if hasattr(event, "auth_events") and event.auth_events: - auth_ids = zip(*event.auth_events)[0] + auth_ids = self.hs.get_auth().compute_auth_events( + event, context.current_state + ) context.auth_events = { k: v for k, v in context.current_state.items() @@ -149,7 +151,9 @@ class StateHandler(object): event.unsigned["replaces_state"] = replaces.event_id if hasattr(event, "auth_events") and event.auth_events: - auth_ids = zip(*event.auth_events)[0] + auth_ids = self.hs.get_auth().compute_auth_events( + event, context.current_state + ) context.auth_events = { k: v for k, v in context.current_state.items() -- cgit 1.4.1 From 95e2d2d36d6dfb7205c40fa8e59ef350f8096395 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 15:02:23 +0000 Subject: When returning lists of servers from alias lookups, put the current server first in the list --- synapse/handlers/directory.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 58e9a91562..7b60921040 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -113,7 +113,16 @@ class DirectoryHandler(BaseHandler): ) extra_servers = yield self.store.get_joined_hosts_for_room(room_id) - servers = list(set(extra_servers) | set(servers)) + servers = set(extra_servers) | set(servers) + + # If this server is in the list of servers, return it first. + if self.server_name in servers: + servers = ( + [self.server_name] + + [s for s in servers if s != self.server_name] + ) + else: + servers = list(servers) defer.returnValue({ "room_id": room_id, -- cgit 1.4.1 From 89f2e8fbdf7965d02426ef17ca6a9490219a2ec4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 15:21:03 +0000 Subject: Fix bug in store defer. Add more unit tests. --- synapse/storage/appservice.py | 18 +++++-- tests/appservice/test_appservice.py | 87 ++++++++++++++++++++++++++++++ tests/handlers/test_appservice.py | 6 +-- tests/storage/test_appservice.py | 105 ++++++++++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 9 deletions(-) create mode 100644 tests/storage/test_appservice.py (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index abb617f049..b64416de28 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -40,7 +40,7 @@ class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.cache = ApplicationServiceCache() - self._populate_cache() + self.cache_defer = self._populate_cache() @defer.inlineCallbacks def unregister_app_service(self, token): @@ -49,6 +49,7 @@ class ApplicationServiceStore(SQLBaseStore): This removes all AS specific regex and the base URL. The token is the only thing preserved for future registration attempts. """ + yield self.cache_defer # make sure the cache is ready yield self.runInteraction( "unregister_app_service", self._unregister_app_service_txn, @@ -89,9 +90,13 @@ class ApplicationServiceStore(SQLBaseStore): Args: service(ApplicationService): The updated service. """ + yield self.cache_defer # make sure the cache is ready + # NB: There is no "insert" since we provide no public-facing API to # allocate new ASes. It relies on the server admin inserting the AS # token into the database manually. + + if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") @@ -148,9 +153,12 @@ class ApplicationServiceStore(SQLBaseStore): if res: return res[0] + @defer.inlineCallbacks def get_app_services(self): - return self.cache.services + yield self.cache_defer # make sure the cache is ready + defer.returnValue(self.cache.services) + @defer.inlineCallbacks def get_app_service_by_token(self, token, from_cache=True): """Get the application service with the given token. @@ -161,12 +169,14 @@ class ApplicationServiceStore(SQLBaseStore): Raises: StoreError if there was a problem retrieving this service. """ + yield self.cache_defer # make sure the cache is ready if from_cache: for service in self.cache.services: if service.token == token: - return service - return None + defer.returnValue(service) + return + defer.returnValue(None) # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 5cfd26daa6..c0aaf12785 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -56,3 +56,90 @@ class ApplicationServiceTestCase(unittest.TestCase): self.event.type = "m.room.member" self.event.state_key = "@irc_foobar:matrix.org" self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_room_id_match(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + "!some_prefix.*some_suffix:matrix.org" + ) + self.event.room_id = "!some_prefixs0m3th1nGsome_suffix:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_room_id_no_match(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + "!some_prefix.*some_suffix:matrix.org" + ) + self.event.room_id = "!XqBunHwQIXUiqCaoxq:matrix.org" + self.assertFalse(self.service.is_interested(self.event)) + + def test_regex_alias_match(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#irc_.*:matrix.org" + ) + self.assertTrue(self.service.is_interested( + self.event, + aliases_for_event=["#irc_foobar:matrix.org", "#athing:matrix.org"] + )) + + def test_regex_alias_no_match(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#irc_.*:matrix.org" + ) + self.assertFalse(self.service.is_interested( + self.event, + aliases_for_event=["#xmpp_foobar:matrix.org", "#athing:matrix.org"] + )) + + def test_regex_multiple_matches(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#irc_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested( + self.event, + aliases_for_event=["#irc_barfoo:matrix.org"] + )) + + def test_restrict_to_rooms(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + "!flibble_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.event.room_id = "!wibblewoo:matrix.org" + self.assertFalse(self.service.is_interested( + self.event, + restrict_to=ApplicationService.NS_ROOMS + )) + + def test_restrict_to_aliases(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#xmpp_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertFalse(self.service.is_interested( + self.event, + restrict_to=ApplicationService.NS_ALIASES, + aliases_for_event=["#irc_barfoo:matrix.org"] + )) + + def test_restrict_to_senders(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#xmpp_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@xmpp_foobar:matrix.org" + self.assertFalse(self.service.is_interested( + self.event, + restrict_to=ApplicationService.NS_USERS, + aliases_for_event=["#xmpp_barfoo:matrix.org"] + )) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 9c464e7fbc..1daa314f20 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -18,12 +18,8 @@ from .. import unittest from synapse.handlers.appservice import ApplicationServicesHandler -from collections import namedtuple from mock import Mock -# TODO: Should this be a more general thing? tests/api/test_filtering.py uses it -MockEvent = namedtuple("MockEvent", "sender type room_id") - class AppServiceHandlerTestCase(unittest.TestCase): """ Tests the ApplicationServicesHandler. """ @@ -51,7 +47,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): self.mock_store.get_app_services = Mock(return_value=services) - event = MockEvent( + event = Mock( sender="@someone:anywhere", type="m.room.message", room_id="!foo:bar" diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py new file mode 100644 index 0000000000..56fdda377c --- /dev/null +++ b/tests/storage/test_appservice.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from tests import unittest +from twisted.internet import defer + +from synapse.appservice import ApplicationService +from synapse.server import HomeServer +from synapse.storage.appservice import ApplicationServiceStore + +from tests.utils import SQLiteMemoryDbPool, MockClock + + +class ApplicationServiceStoreTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer("test", db_pool=db_pool, clock=MockClock()) + self.as_token = "token1" + db_pool.runQuery( + "INSERT INTO application_services(token) VALUES(?)", + (self.as_token,) + ) + db_pool.runQuery( + "INSERT INTO application_services(token) VALUES(?)", ("token2",) + ) + db_pool.runQuery( + "INSERT INTO application_services(token) VALUES(?)", ("token3",) + ) + # must be done after inserts + self.store = ApplicationServiceStore(hs) + + @defer.inlineCallbacks + def test_update_and_retrieval_of_service(self): + url = "https://matrix.org/appservices/foobar" + user_regex = ["@foobar_.*:matrix.org"] + alias_regex = ["#foobar_.*:matrix.org"] + room_regex = [] + service = ApplicationService(url=url, token=self.as_token, namespaces={ + ApplicationService.NS_USERS: user_regex, + ApplicationService.NS_ALIASES: alias_regex, + ApplicationService.NS_ROOMS: room_regex + }) + yield self.store.update_app_service(service) + + stored_service = yield self.store.get_app_service_by_token( + self.as_token + ) + self.assertEquals(stored_service.token, self.as_token) + self.assertEquals(stored_service.url, url) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ALIASES], + alias_regex + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ROOMS], + room_regex + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_USERS], + user_regex + ) + + @defer.inlineCallbacks + def test_retrieve_unknown_service_token(self): + service = yield self.store.get_app_service_by_token("invalid_token") + self.assertEquals(service, None) + + @defer.inlineCallbacks + def test_retrieval_of_service(self): + stored_service = yield self.store.get_app_service_by_token( + self.as_token + ) + self.assertEquals(stored_service.token, self.as_token) + self.assertEquals(stored_service.url, None) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ALIASES], + [] + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ROOMS], + [] + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_USERS], + [] + ) + + @defer.inlineCallbacks + def test_retrieval_of_all_services(self): + services = yield self.store.get_app_services() + self.assertEquals(len(services), 3) -- cgit 1.4.1 From 2e77ba637a75adcf0681e04c281f4fea74ebec6b Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 Feb 2015 16:24:15 +0000 Subject: More s/instance_handle/profile_tag/ --- synapse/rest/client/v1/push_rule.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 348adb9c0d..5582f33c8e 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -73,7 +73,7 @@ class PushRuleRestServlet(ClientV1RestServlet): 'rule_id': rule_id } if device: - spec['device'] = device + spec['profile_tag'] = device return spec def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj, device=None): @@ -188,15 +188,15 @@ class PushRuleRestServlet(ClientV1RestServlet): user, _ = yield self.auth.get_user_by_req(request) - if 'device' in spec: + if 'profile_tag' in spec: rules = yield self.hs.get_datastore().get_push_rules_for_user_name( user.to_string() ) for r in rules: conditions = json.loads(r['conditions']) - ih = _profile_tag_from_conditions(conditions) - if ih == spec['device'] and r['priority_class'] == priority_class: + pt = _profile_tag_from_conditions(conditions) + if pt == spec['profile_tag'] and r['priority_class'] == priority_class: yield self.hs.get_datastore().delete_push_rule( user.to_string(), spec['rule_id'] ) @@ -306,7 +306,7 @@ def _add_empty_priority_class_arrays(d): def _profile_tag_from_conditions(conditions): """ - Given a list of conditions, return the instance handle of the + Given a list of conditions, return the profile tag of the device rule if there is one """ for c in conditions: -- cgit 1.4.1 From ae46f10fc5dba0f81518e3144ab8d9ed7a7d03bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 16:28:12 +0000 Subject: Apply sanity to the transport client interface. Convert 'make_join' and 'send_join' to accept iterables of destinations --- synapse/api/errors.py | 8 +++- synapse/federation/federation_client.py | 82 ++++++++++++++++++++------------- synapse/federation/transaction_queue.py | 23 +++++++-- synapse/federation/transport/client.py | 42 +++++++---------- synapse/handlers/federation.py | 4 +- synapse/http/matrixfederationclient.py | 42 ++++++++++++++--- 6 files changed, 130 insertions(+), 71 deletions(-) (limited to 'synapse') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index ad478aa6b7..5041828f18 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -39,7 +39,7 @@ class Codes(object): TOO_LARGE = "M_TOO_LARGE" -class CodeMessageException(Exception): +class CodeMessageException(RuntimeError): """An exception with integer code and message string attributes.""" def __init__(self, code, msg): @@ -227,3 +227,9 @@ class FederationError(RuntimeError): "affected": self.affected, "source": self.source if self.source else self.affected, } + + +class HttpResponseException(CodeMessageException): + def __init__(self, code, msg, response): + self.response = response + super(HttpResponseException, self).__init__(code, msg) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index d6b8c43916..eb36ec040b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -19,6 +19,7 @@ from twisted.internet import defer from .federation_base import FederationBase from .units import Edu +from synapse.api.errors import CodeMessageException from synapse.util.logutils import log_function from synapse.events import FrozenEvent @@ -180,7 +181,8 @@ class FederationClient(FederationBase): pdu = yield self._check_sigs_and_hash(pdu) break - + except CodeMessageException: + raise except Exception as e: logger.info( "Failed to get PDU %s from %s because %s", @@ -264,45 +266,63 @@ class FederationClient(FederationBase): defer.returnValue(self.event_from_pdu_json(pdu_dict)) break - except Exception as e: - logger.warn("Failed to make_join via %s", destination) + except CodeMessageException: + raise + except RuntimeError as e: + logger.warn( + "Failed to make_join via %s: %s", + destination, e.message + ) + + raise RuntimeError("Failed to send to any server.") @defer.inlineCallbacks - def send_join(self, destination, pdu): - time_now = self._clock.time_msec() - _, content = yield self.transport_layer.send_join( - destination=destination, - room_id=pdu.room_id, - event_id=pdu.event_id, - content=pdu.get_pdu_json(time_now), - ) + def send_join(self, destinations, pdu): + for destination in destinations: + try: + time_now = self._clock.time_msec() + _, content = yield self.transport_layer.send_join( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=pdu.get_pdu_json(time_now), + ) - logger.debug("Got content: %s", content) + logger.debug("Got content: %s", content) - state = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("state", []) - ] + state = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("state", []) + ] - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("auth_chain", []) - ] + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("auth_chain", []) + ] - signed_state = yield self._check_sigs_and_hash_and_fetch( - destination, state, outlier=True - ) + signed_state = yield self._check_sigs_and_hash_and_fetch( + destination, state, outlier=True + ) - signed_auth = yield self._check_sigs_and_hash_and_fetch( - destination, auth_chain, outlier=True - ) + signed_auth = yield self._check_sigs_and_hash_and_fetch( + destination, auth_chain, outlier=True + ) - auth_chain.sort(key=lambda e: e.depth) + auth_chain.sort(key=lambda e: e.depth) + + defer.returnValue({ + "state": signed_state, + "auth_chain": signed_auth, + }) + except CodeMessageException: + raise + except RuntimeError as e: + logger.warn( + "Failed to send_join via %s: %s", + destination, e.message + ) - defer.returnValue({ - "state": signed_state, - "auth_chain": signed_auth, - }) + raise RuntimeError("Failed to send to any server.") @defer.inlineCallbacks def send_invite(self, destination, room_id, event_id, pdu): diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 9d4f2c09a2..f38aeba7cc 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -19,6 +19,7 @@ from twisted.internet import defer from .persistence import TransactionActions from .units import Transaction +from synapse.api.errors import HttpResponseException from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext @@ -238,9 +239,14 @@ class TransactionQueue(object): del p["age_ts"] return data - code, response = yield self.transport_layer.send_transaction( - transaction, json_data_cb - ) + try: + response = yield self.transport_layer.send_transaction( + transaction, json_data_cb + ) + code = 200 + except HttpResponseException as e: + code = e.code + response = e.response logger.info("TX [%s] got %d response", destination, code) @@ -274,8 +280,7 @@ class TransactionQueue(object): pass logger.debug("TX [%s] Yielded to callbacks", destination) - - except Exception as e: + except RuntimeError as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. logger.warn( @@ -283,6 +288,14 @@ class TransactionQueue(object): destination, e, ) + except Exception as e: + # We capture this here as there as nothing actually listens + # for this finishing functions deferred. + logger.exception( + "TX [%s] Problem in _attempt_transaction: %s", + destination, + e, + ) self.set_retrying(destination, retry_interval) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 4cb1dea2de..8b137e7128 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -19,7 +19,6 @@ from synapse.api.urls import FEDERATION_PREFIX as PREFIX from synapse.util.logutils import log_function import logging -import json logger = logging.getLogger(__name__) @@ -129,7 +128,7 @@ class TransportLayerClient(object): # generated by the json_data_callback. json_data = transaction.get_dict() - code, response = yield self.client.put_json( + response = yield self.client.put_json( transaction.destination, path=PREFIX + "/send/%s/" % transaction.transaction_id, data=json_data, @@ -137,95 +136,86 @@ class TransportLayerClient(object): ) logger.debug( - "send_data dest=%s, txid=%s, got response: %d", - transaction.destination, transaction.transaction_id, code + "send_data dest=%s, txid=%s, got response: 200", + transaction.destination, transaction.transaction_id, ) - defer.returnValue((code, response)) + defer.returnValue(response) @defer.inlineCallbacks @log_function def make_query(self, destination, query_type, args, retry_on_dns_fail): path = PREFIX + "/query/%s" % query_type - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, args=args, retry_on_dns_fail=retry_on_dns_fail, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def make_join(self, destination, room_id, user_id, retry_on_dns_fail=True): path = PREFIX + "/make_join/%s/%s" % (room_id, user_id) - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, retry_on_dns_fail=retry_on_dns_fail, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def send_join(self, destination, room_id, event_id, content): path = PREFIX + "/send_join/%s/%s" % (room_id, event_id) - code, content = yield self.client.put_json( + response = yield self.client.put_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_join", code) - - defer.returnValue(json.loads(content)) + defer.returnValue(response) @defer.inlineCallbacks @log_function def send_invite(self, destination, room_id, event_id, content): path = PREFIX + "/invite/%s/%s" % (room_id, event_id) - code, content = yield self.client.put_json( + response = yield self.client.put_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_invite", code) - - defer.returnValue(json.loads(content)) + defer.returnValue(response) @defer.inlineCallbacks @log_function def get_event_auth(self, destination, room_id, event_id): path = PREFIX + "/event_auth/%s/%s" % (room_id, event_id) - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def send_query_auth(self, destination, room_id, event_id, content): path = PREFIX + "/query_auth/%s/%s" % (room_id, event_id) - code, content = yield self.client.post_json( + content = yield self.client.post_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_invite", code) - - defer.returnValue(json.loads(content)) + defer.returnValue(content) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0876589e31..a968a87360 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -288,7 +288,7 @@ class FederationHandler(BaseHandler): logger.debug("Joining %s to %s", joinee, room_id) pdu = yield self.replication_layer.make_join( - target_host, + [target_host], room_id, joinee ) @@ -331,7 +331,7 @@ class FederationHandler(BaseHandler): new_event = builder.build() ret = yield self.replication_layer.send_join( - target_host, + [target_host], new_event ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index c7bf1b47b8..8559d06b7f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -27,7 +27,9 @@ from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json -from synapse.api.errors import CodeMessageException, SynapseError, Codes +from synapse.api.errors import ( + SynapseError, Codes, HttpResponseException, +) from syutil.crypto.jsonsign import sign_json @@ -163,13 +165,12 @@ class MatrixFederationHttpClient(object): ) if 200 <= response.code < 300: - # We need to update the transactions table to say it was sent? pass else: # :'( # Update transactions table? - raise CodeMessageException( - response.code, response.phrase + raise HttpResponseException( + response.code, response.phrase, response ) defer.returnValue(response) @@ -238,11 +239,20 @@ class MatrixFederationHttpClient(object): headers_dict={"Content-Type": ["application/json"]}, ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + logger.debug("Getting resp body") body = yield readBody(response) logger.debug("Got resp body") - defer.returnValue((response.code, body)) + defer.returnValue(json.loads(body)) @defer.inlineCallbacks def post_json(self, destination, path, data={}): @@ -275,11 +285,20 @@ class MatrixFederationHttpClient(object): headers_dict={"Content-Type": ["application/json"]}, ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + logger.debug("Getting resp body") body = yield readBody(response) logger.debug("Got resp body") - defer.returnValue((response.code, body)) + defer.returnValue(json.loads(body)) @defer.inlineCallbacks def get_json(self, destination, path, args={}, retry_on_dns_fail=True): @@ -321,7 +340,18 @@ class MatrixFederationHttpClient(object): retry_on_dns_fail=retry_on_dns_fail ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + + logger.debug("Getting resp body") body = yield readBody(response) + logger.debug("Got resp body") defer.returnValue(json.loads(body)) -- cgit 1.4.1 From aa8cce58bf3d3dbd1a0d512dbbd41b6545d2b1f9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 16:44:53 +0000 Subject: Add query_user/alias APIs. --- synapse/appservice/api.py | 47 ++++++++++++++++++++++++++++++++++++++---- synapse/handlers/appservice.py | 26 ++++++++++++++--------- 2 files changed, 59 insertions(+), 14 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 158aded66e..799ada96df 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -12,25 +12,64 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer +from twisted.web.client import PartialDownloadError +from synapse.http.client import SimpleHttpClient -class ApplicationServiceApi(object): +import logging +import urllib + +logger = logging.getLogger(__name__) + + +class ApplicationServiceApi(SimpleHttpClient): """This class manages HS -> AS communications, including querying and pushing. """ def __init__(self, hs): + super(ApplicationServiceApi, self).__init__(hs) self.hs_token = "_hs_token_" # TODO extract hs token + @defer.inlineCallbacks def query_user(self, service, user_id): - pass + uri = service.url + ("/users/%s" % urllib.quote(user_id)) + response = None + try: + response = yield self.get_json(uri, { + "access_token": self.hs_token + }) + if response: # just an empty json object + defer.returnValue(True) + except PartialDownloadError as e: + if e.status == 404: + defer.returnValue(False) + return + logger.warning("query_user to %s received %s", (uri, e.status)) + @defer.inlineCallbacks def query_alias(self, service, alias): - pass + uri = service.url + ("/rooms/%s" % urllib.quote(alias)) + response = None + try: + response = yield self.get_json(uri, { + "access_token": self.hs_token + }) + logger.info("%s", response[0]) + if response: # just an empty json object + defer.returnValue(True) + except PartialDownloadError as e: + if e.status == 404: + defer.returnValue(False) + return + logger.warning("query_alias to %s received %s", (uri, e.status)) def push_bulk(self, service, events): pass + @defer.inlineCallbacks def push(self, service, event): - pass + response = yield self.push_bulk(service, [event]) + defer.returnValue(response) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 3188c60f3d..dd860b2244 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -55,10 +55,14 @@ class ApplicationServicesHandler(object): logger.info("Updating application service info...") yield self.store.update_app_service(app_service) + logger.info("Sending ping to %s...", app_service.url) + yield self.appservice_api.query_alias(app_service, "ping") + def unregister(self, token): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) + @defer.inlineCallbacks def get_services_for_event(self, event, restrict_to=""): """Retrieve a list of application services interested in this event. @@ -71,14 +75,15 @@ class ApplicationServicesHandler(object): """ # We need to know the aliases associated with this event.room_id, if any alias_list = [] # TODO - + services = yield self.store.get_app_services() interested_list = [ - s for s in self.store.get_app_services() if ( + s for s in services if ( s.is_interested(event, restrict_to, alias_list) ) ] - return interested_list + defer.returnValue(interested_list) + @defer.inlineCallbacks def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. @@ -89,7 +94,7 @@ class ApplicationServicesHandler(object): event(Event): The event to push out to interested services. """ # Gather interested services - services = self.get_services_for_event(event) + services = yield self.get_services_for_event(event) if len(services) == 0: return # no services need notifying @@ -97,14 +102,14 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = False # TODO check if unknown_user: - user_query_services = self.get_services_for_event( + user_query_services = yield self.get_services_for_event( event=event, restrict_to=ApplicationService.NS_USERS ) for user_service in user_query_services: # this needs to block XXX: Need to feed response back to caller - is_known_user = self.appservice_api.query_user( - user_service, event + is_known_user = yield self.appservice_api.query_user( + user_service, event.sender ) if is_known_user: # the user exists now,so don't query more ASes. @@ -114,14 +119,15 @@ class ApplicationServicesHandler(object): # API for all services which match that room alias regex. unknown_room_alias = False # TODO check if unknown_room_alias: - alias_query_services = self.get_services_for_event( + alias = "something" # TODO + alias_query_services = yield self.get_services_for_event( event=event, restrict_to=ApplicationService.NS_ALIASES ) for alias_service in alias_query_services: # this needs to block XXX: Need to feed response back to caller - is_known_alias = self.appservice_api.query_alias( - alias_service, event + is_known_alias = yield self.appservice_api.query_alias( + alias_service, alias ) if is_known_alias: # the alias exists now so don't query more ASes. -- cgit 1.4.1 From 96d4bf90120a07faa5163c2e5af542358554dd61 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:07:31 +0000 Subject: Modify API for SimpleHttpClient.get_json and update usages. Previously, this would only return the HTTP body as JSON, and discard other response information (e.g. the HTTP response code). This has now been changed to throw a CodeMessageException on a non-2xx response, with the response code and body, which can then be parsed as JSON. Affected modules include: - Registration/Login (when using an email for IS auth) --- synapse/handlers/login.py | 33 +++++++++++++++----------- synapse/handlers/register.py | 56 ++++++++++++++++++++++++++------------------ synapse/http/client.py | 19 +++++++-------- 3 files changed, 61 insertions(+), 47 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index d297d71c03..7447800460 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -16,12 +16,13 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.errors import LoginError, Codes +from synapse.api.errors import LoginError, Codes, CodeMessageException from synapse.http.client import SimpleHttpClient from synapse.util.emailutils import EmailException import synapse.util.emailutils as emailutils import bcrypt +import json import logging logger = logging.getLogger(__name__) @@ -96,16 +97,20 @@ class LoginHandler(BaseHandler): @defer.inlineCallbacks def _query_email(self, email): - httpCli = SimpleHttpClient(self.hs) - data = yield httpCli.get_json( - # TODO FIXME This should be configurable. - # XXX: ID servers need to use HTTPS - "http://%s%s" % ( - "matrix.org:8090", "/_matrix/identity/api/v1/lookup" - ), - { - 'medium': 'email', - 'address': email - } - ) - defer.returnValue(data) + http_client = SimpleHttpClient(self.hs) + try: + data = yield http_client.get_json( + # TODO FIXME This should be configurable. + # XXX: ID servers need to use HTTPS + "http://%s%s" % ( + "matrix.org:8090", "/_matrix/identity/api/v1/lookup" + ), + { + 'medium': 'email', + 'address': email + } + ) + defer.returnValue(data) + except CodeMessageException as e: + data = json.loads(e.msg) + defer.returnValue(data) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 66a89c10b2..08cd5fd720 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.types import UserID from synapse.api.errors import ( - SynapseError, RegistrationError, InvalidCaptchaError + SynapseError, RegistrationError, InvalidCaptchaError, CodeMessageException ) from ._base import BaseHandler import synapse.util.stringutils as stringutils @@ -28,6 +28,7 @@ from synapse.http.client import CaptchaServerHttpClient import base64 import bcrypt +import json import logging logger = logging.getLogger(__name__) @@ -161,21 +162,26 @@ class RegistrationHandler(BaseHandler): def _threepid_from_creds(self, creds): # TODO: get this from the homeserver rather than creating a new one for # each request - httpCli = SimpleHttpClient(self.hs) + http_client = SimpleHttpClient(self.hs) # XXX: make this configurable! trustedIdServers = ['matrix.org:8090', 'matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) defer.returnValue(None) - data = yield httpCli.get_json( - # XXX: This should be HTTPS - "http://%s%s" % ( - creds['idServer'], - "/_matrix/identity/api/v1/3pid/getValidated3pid" - ), - {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} - ) + + data = {} + try: + data = yield http_client.get_json( + # XXX: This should be HTTPS + "http://%s%s" % ( + creds['idServer'], + "/_matrix/identity/api/v1/3pid/getValidated3pid" + ), + {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} + ) + except CodeMessageException as e: + data = json.loads(e.msg) if 'medium' in data: defer.returnValue(data) @@ -185,19 +191,23 @@ class RegistrationHandler(BaseHandler): def _bind_threepid(self, creds, mxid): yield logger.debug("binding threepid") - httpCli = SimpleHttpClient(self.hs) - data = yield httpCli.post_urlencoded_get_json( - # XXX: Change when ID servers are all HTTPS - "http://%s%s" % ( - creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" - ), - { - 'sid': creds['sid'], - 'clientSecret': creds['clientSecret'], - 'mxid': mxid, - } - ) - logger.debug("bound threepid") + http_client = SimpleHttpClient(self.hs) + data = None + try: + data = yield http_client.post_urlencoded_get_json( + # XXX: Change when ID servers are all HTTPS + "http://%s%s" % ( + creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" + ), + { + 'sid': creds['sid'], + 'clientSecret': creds['clientSecret'], + 'mxid': mxid, + } + ) + logger.debug("bound threepid") + except CodeMessageException as e: + data = json.loads(e.msg) defer.returnValue(data) @defer.inlineCallbacks diff --git a/synapse/http/client.py b/synapse/http/client.py index 198f575cfa..5f4558be47 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from synapse.api.errors import CodeMessageException from synapse.http.agent_name import AGENT_NAME from twisted.internet import defer, reactor from twisted.web.client import ( @@ -83,7 +83,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def get_json(self, uri, args={}): - """ Get's some json from the given host and path + """ Gets some json from the given host and path Args: uri (str): The URI to request, not including query parameters @@ -91,15 +91,11 @@ class SimpleHttpClient(object): None. **Note**: The value of each key is assumed to be an iterable and *not* a string. - Returns: - Deferred: Succeeds when we get *any* HTTP response. - - The result of the deferred is a tuple of `(code, response)`, - where `response` is a dict representing the decoded JSON body. + Deferred: Succeeds when we get *any* 2xx HTTP response. + Raises: + On a non-2xx HTTP response. """ - - yield if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) @@ -114,7 +110,10 @@ class SimpleHttpClient(object): body = yield readBody(response) - defer.returnValue(json.loads(body)) + if 200 <= response.code < 300: + defer.returnValue(json.loads(body)) + else: + raise CodeMessageException(response.code, body) class CaptchaServerHttpClient(SimpleHttpClient): -- cgit 1.4.1 From 6d3e4f4d0aad4ad9b44b28349838ff48aef39440 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:32:44 +0000 Subject: Update user/alias query APIs to use new format of SimpleHttpClient.get_json --- synapse/appservice/api.py | 15 +++++++-------- synapse/http/client.py | 3 +++ 2 files changed, 10 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 799ada96df..74508ecddf 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from twisted.internet import defer -from twisted.web.client import PartialDownloadError +from synapse.api.errors import CodeMessageException from synapse.http.client import SimpleHttpClient import logging @@ -42,11 +42,11 @@ class ApplicationServiceApi(SimpleHttpClient): }) if response: # just an empty json object defer.returnValue(True) - except PartialDownloadError as e: - if e.status == 404: + except CodeMessageException as e: + if e.code == 404: defer.returnValue(False) return - logger.warning("query_user to %s received %s", (uri, e.status)) + logger.warning("query_user to %s received %s", uri, e.code) @defer.inlineCallbacks def query_alias(self, service, alias): @@ -56,14 +56,13 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": self.hs_token }) - logger.info("%s", response[0]) if response: # just an empty json object defer.returnValue(True) - except PartialDownloadError as e: - if e.status == 404: + except CodeMessageException as e: + if e.code == 404: defer.returnValue(False) return - logger.warning("query_alias to %s received %s", (uri, e.status)) + logger.warning("query_alias to %s received %s", uri, e.code) def push_bulk(self, service, events): pass diff --git a/synapse/http/client.py b/synapse/http/client.py index 5f4558be47..fee8c901a2 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -113,6 +113,9 @@ class SimpleHttpClient(object): if 200 <= response.code < 300: defer.returnValue(json.loads(body)) else: + # NB: This is explicitly not json.loads(body)'d because the contract + # of CodeMessageException is a *string* message. Callers can always + # load it into JSON if they want. raise CodeMessageException(response.code, body) -- cgit 1.4.1 From 543e84fe70bdc0af6be5feb237de2d18adf352ab Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:39:51 +0000 Subject: Add SimpleHttpClient.put_json with the same semantics as get_json. --- synapse/http/client.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/http/client.py b/synapse/http/client.py index fee8c901a2..575510637e 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -83,7 +83,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def get_json(self, uri, args={}): - """ Gets some json from the given host and path + """ Gets some json from the given URI. Args: uri (str): The URI to request, not including query parameters @@ -92,7 +92,8 @@ class SimpleHttpClient(object): **Note**: The value of each key is assumed to be an iterable and *not* a string. Returns: - Deferred: Succeeds when we get *any* 2xx HTTP response. + Deferred: Succeeds when we get *any* 2xx HTTP response, with the + HTTP body as JSON. Raises: On a non-2xx HTTP response. """ @@ -118,6 +119,49 @@ class SimpleHttpClient(object): # load it into JSON if they want. raise CodeMessageException(response.code, body) + @defer.inlineCallbacks + def put_json(self, uri, json_body, args={}): + """ Puts some json to the given URI. + + Args: + uri (str): The URI to request, not including query parameters + json_body (dict): The JSON to put in the HTTP body, + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + Returns: + Deferred: Succeeds when we get *any* 2xx HTTP response, with the + HTTP body as JSON. + Raises: + On a non-2xx HTTP response. + """ + if len(args): + query_bytes = urllib.urlencode(args, True) + uri = "%s?%s" % (uri, query_bytes) + + json_str = json.dumps(json_body) + + response = yield self.agent.request( + "PUT", + uri.encode("ascii"), + headers=Headers({ + b"User-Agent": [AGENT_NAME], + "Content-Type": ["application/json"] + }), + bodyProducer=FileBodyProducer(StringIO(json_str)) + ) + + body = yield readBody(response) + + if 200 <= response.code < 300: + defer.returnValue(json.loads(body)) + else: + # NB: This is explicitly not json.loads(body)'d because the contract + # of CodeMessageException is a *string* message. Callers can always + # load it into JSON if they want. + raise CodeMessageException(response.code, body) + class CaptchaServerHttpClient(SimpleHttpClient): """ -- cgit 1.4.1 From 19ebdc321d07e7f4a2946a284885e45e5434cb44 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 23:51:02 +0000 Subject: Pull in python_dependencies.py from develop --- synapse/python_dependencies.py | 122 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 synapse/python_dependencies.py (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py new file mode 100644 index 0000000000..a89d618606 --- /dev/null +++ b/synapse/python_dependencies.py @@ -0,0 +1,122 @@ +import logging +from distutils.version import LooseVersion + +logger = logging.getLogger(__name__) + +REQUIREMENTS = { + "syutil==0.0.2": ["syutil"], + "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], + "Twisted==14.0.2": ["twisted==14.0.2"], + "service_identity>=1.0.0": ["service_identity>=1.0.0"], + "pyopenssl>=0.14": ["OpenSSL>=0.14"], + "pyyaml": ["yaml"], + "pyasn1": ["pyasn1"], + "pynacl": ["nacl"], + "daemonize": ["daemonize"], + "py-bcrypt": ["bcrypt"], + "frozendict>=0.4": ["frozendict"], + "pillow": ["PIL"], + "pydenticon": ["pydenticon"], +} + +def github_link(project, version, egg): + return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) + +DEPENDENCY_LINKS=[ + github_link( + project="matrix-org/syutil", + version="v0.0.2", + egg="syutil-0.0.2", + ), + github_link( + project="matrix-org/matrix-angular-sdk", + version="v0.6.0", + egg="matrix_angular_sdk-0.6.0", + ), + github_link( + project="pyca/pynacl", + version="d4d3175589b892f6ea7c22f466e0e223853516fa", + egg="pynacl-0.3.0", + ) +] + + +class MissingRequirementError(Exception): + pass + + +def check_requirements(): + """Checks that all the modules needed by synapse have been correctly + installed and are at the correct version""" + for dependency, module_requirements in REQUIREMENTS.items(): + for module_requirement in module_requirements: + if ">=" in module_requirement: + module_name, required_version = module_requirement.split(">=") + version_test = ">=" + elif "==" in module_requirement: + module_name, required_version = module_requirement.split("==") + version_test = "==" + else: + module_name = module_requirement + version_test = None + + try: + module = __import__(module_name) + except ImportError: + logging.exception( + "Can't import %r which is part of %r", + module_name, dependency + ) + raise MissingRequirementError( + "Can't import %r which is part of %r" + % (module_name, dependency) + ) + version = getattr(module, "__version__", None) + file_path = getattr(module, "__file__", None) + logger.info( + "Using %r version %r from %r to satisfy %r", + module_name, version, file_path, dependency + ) + + if version_test == ">=": + if version is None: + raise MissingRequirementError( + "Version of %r isn't set as __version__ of module %r" + % (dependency, module_name) + ) + if LooseVersion(version) < LooseVersion(required_version): + raise MissingRequirementError( + "Version of %r in %r is too old. %r < %r" + % (dependency, file_path, version, required_version) + ) + elif version_test == "==": + if version is None: + raise MissingRequirementError( + "Version of %r isn't set as __version__ of module %r" + % (dependency, module_name) + ) + if LooseVersion(version) != LooseVersion(required_version): + raise MissingRequirementError( + "Unexpected version of %r in %r. %r != %r" + % (dependency, file_path, version, required_version) + ) + +def list_requirements(): + result = [] + linked = [] + for link in DEPENDENCY_LINKS: + egg = link.split("#egg=")[1] + linked.append(egg.split('-')[0]) + result.append(link) + for requirement in REQUIREMENTS: + is_linked = False + for link in linked: + if requirement.replace('-','_').startswith(link): + is_linked = True + if not is_linked: + result.append(requirement) + return result + +if __name__ == "__main__": + import sys + sys.stdout.writelines(req + "\n" for req in list_requirements()) -- cgit 1.4.1 From 77e5ae22a99e6668e76b530735e594509dcc3ea7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 23:51:34 +0000 Subject: Ver bump --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/VERSION b/VERSION index 1c29ff4d36..1648b8029d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1d +0.6.1e diff --git a/synapse/__init__.py b/synapse/__init__.py index dacda89865..d3f1c33d3a 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.1d" +__version__ = "0.6.1e" -- cgit 1.4.1 From a1a4960baf78c1a24f76c603076c832a1947360f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 09:43:22 +0000 Subject: Impl push_bulk function --- synapse/appservice/api.py | 19 ++++++++++++++++++- synapse/handlers/appservice.py | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 74508ecddf..fbf4abc526 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -64,8 +64,25 @@ class ApplicationServiceApi(SimpleHttpClient): return logger.warning("query_alias to %s received %s", uri, e.code) + @defer.inlineCallbacks def push_bulk(self, service, events): - pass + uri = service.url + ("/transactions/%s" % + urllib.quote(str(0))) # TODO txn_ids + response = None + try: + response = yield self.put_json( + uri, + { + "events": events + }, + { + "access_token": self.hs_token + }) + if response: # just an empty json object + defer.returnValue(True) + except CodeMessageException as e: + logger.warning("push_bulk to %s received %s", uri, e.code) + defer.returnValue(False) @defer.inlineCallbacks def push(self, service, event): diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index dd860b2244..2b2761682f 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -56,7 +56,7 @@ class ApplicationServicesHandler(object): yield self.store.update_app_service(app_service) logger.info("Sending ping to %s...", app_service.url) - yield self.appservice_api.query_alias(app_service, "ping") + yield self.appservice_api.push(app_service, "pinger") def unregister(self, token): logger.info("Unregister as_token=%s", token) -- cgit 1.4.1 From 27091f146a0ebdbfe1ae7c5cd30de51515cfbebc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 10:08:12 +0000 Subject: Add hs_token column and generate a different token f.e application service. --- synapse/appservice/__init__.py | 6 ++++-- synapse/appservice/api.py | 8 ++++---- synapse/handlers/appservice.py | 9 ++++++--- synapse/rest/appservice/v1/register.py | 4 ++-- synapse/storage/appservice.py | 17 ++++++++++++----- synapse/storage/schema/application_services.sql | 1 + tests/storage/test_appservice.py | 10 ++++++---- 7 files changed, 35 insertions(+), 20 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 0c7f58574e..f7baf578f0 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -34,11 +34,13 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - def __init__(self, token, url=None, namespaces=None, txn_id=None): + def __init__(self, token, url=None, namespaces=None, hs_token=None, + txn_id=None): self.token = token self.url = url + self.hs_token = hs_token self.namespaces = self._check_namespaces(namespaces) - self.txn_id = None + self.txn_id = txn_id def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index fbf4abc526..29bb35d61b 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -30,7 +30,6 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) - self.hs_token = "_hs_token_" # TODO extract hs token @defer.inlineCallbacks def query_user(self, service, user_id): @@ -38,7 +37,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.get_json(uri, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object defer.returnValue(True) @@ -54,7 +53,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.get_json(uri, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object defer.returnValue(True) @@ -76,9 +75,10 @@ class ApplicationServiceApi(SimpleHttpClient): "events": events }, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object + # TODO: Mark txn as sent successfully defer.returnValue(True) except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2b2761682f..7b0599c71e 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -19,6 +19,7 @@ from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi +import synapse.util.stringutils as stringutils import logging @@ -53,10 +54,9 @@ class ApplicationServicesHandler(object): errcode=Codes.FORBIDDEN ) logger.info("Updating application service info...") + app_service.hs_token = self._generate_hs_token() yield self.store.update_app_service(app_service) - - logger.info("Sending ping to %s...", app_service.url) - yield self.appservice_api.push(app_service, "pinger") + defer.returnValue(app_service) def unregister(self, token): logger.info("Unregister as_token=%s", token) @@ -136,3 +136,6 @@ class ApplicationServicesHandler(object): # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + + def _generate_hs_token(self): + return stringutils.random_string(18) diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index e374d538e7..d3d5aef220 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -61,8 +61,8 @@ class RegisterRestServlet(AppServiceRestServlet): app_service = ApplicationService(as_token, as_url, namespaces) - yield self.handler.register(app_service) - hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? + app_service = yield self.handler.register(app_service) + hs_token = app_service.hs_token defer.returnValue((200, { "hs_token": hs_token diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index b64416de28..3c8bf9ad0d 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -60,6 +60,7 @@ class ApplicationServiceStore(SQLBaseStore): if service.token == token: service.url = None service.namespaces = None + service.hs_token = None def _unregister_app_service_txn(self, txn, token): # kill the url to prevent pushes @@ -100,6 +101,9 @@ class ApplicationServiceStore(SQLBaseStore): if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") + if not service.hs_token: + raise StoreError(500, "No HS token") + yield self.runInteraction( "update_app_service", self._update_app_service_txn, @@ -126,8 +130,8 @@ class ApplicationServiceStore(SQLBaseStore): return False txn.execute( - "UPDATE application_services SET url=? WHERE id=?", - (service.url, as_id,) + "UPDATE application_services SET url=?, hs_token=? WHERE id=?", + (service.url, service.hs_token, as_id,) ) # cleanup regex txn.execute( @@ -196,6 +200,7 @@ class ApplicationServiceStore(SQLBaseStore): # 'namespace': enum, # 'as_id': 0, # 'token': "something", + # 'hs_token': "otherthing", # 'id': 0 # } # ] @@ -208,6 +213,7 @@ class ApplicationServiceStore(SQLBaseStore): services[as_token] = { "url": res["url"], "token": as_token, + "hs_token": res["hs_token"], "namespaces": { ApplicationService.NS_USERS: [], ApplicationService.NS_ALIASES: [], @@ -230,8 +236,9 @@ class ApplicationServiceStore(SQLBaseStore): for service in services.values(): logger.info("Found application service: %s", service) self.cache.services.append(ApplicationService( - service["token"], - service["url"], - service["namespaces"] + token=service["token"], + url=service["url"], + namespaces=service["namespaces"], + hs_token=service["hs_token"] )) diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql index 6d245fc807..03b5a10c8a 100644 --- a/synapse/storage/schema/application_services.sql +++ b/synapse/storage/schema/application_services.sql @@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS application_services( id INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT, token TEXT, + hs_token TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 56fdda377c..b9ecfb3384 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -46,13 +46,15 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_update_and_retrieval_of_service(self): url = "https://matrix.org/appservices/foobar" + hs_token = "hstok" user_regex = ["@foobar_.*:matrix.org"] alias_regex = ["#foobar_.*:matrix.org"] room_regex = [] - service = ApplicationService(url=url, token=self.as_token, namespaces={ - ApplicationService.NS_USERS: user_regex, - ApplicationService.NS_ALIASES: alias_regex, - ApplicationService.NS_ROOMS: room_regex + service = ApplicationService( + url=url, hs_token=hs_token, token=self.as_token, namespaces={ + ApplicationService.NS_USERS: user_regex, + ApplicationService.NS_ALIASES: alias_regex, + ApplicationService.NS_ROOMS: room_regex }) yield self.store.update_app_service(service) -- cgit 1.4.1 From f0c730252fcb198ad925d6406f0f37ccee29e720 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 11:25:32 +0000 Subject: Add unknown user ID check. Use store.get_aliases_for_room(room_id) when searching for services by alias. --- synapse/handlers/appservice.py | 24 +++++++++++++++++++----- tests/handlers/test_appservice.py | 1 + 2 files changed, 20 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 7b0599c71e..39976b9629 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,10 +15,10 @@ from twisted.internet import defer -from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi +from synapse.types import UserID import synapse.util.stringutils as stringutils import logging @@ -74,7 +74,7 @@ class ApplicationServicesHandler(object): event based on the service regex. """ # We need to know the aliases associated with this event.room_id, if any - alias_list = [] # TODO + alias_list = yield self.store.get_aliases_for_room(event.room_id) services = yield self.store.get_app_services() interested_list = [ s for s in services if ( @@ -100,7 +100,7 @@ class ApplicationServicesHandler(object): # Do we know this user exists? If not, poke the user query API for # all services which match that user regex. - unknown_user = False # TODO check + unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: user_query_services = yield self.get_services_for_event( event=event, @@ -117,7 +117,7 @@ class ApplicationServicesHandler(object): # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. - unknown_room_alias = False # TODO check + unknown_room_alias = False # TODO if unknown_room_alias: alias = "something" # TODO alias_query_services = yield self.get_services_for_event( @@ -137,5 +137,19 @@ class ApplicationServicesHandler(object): for service in services: self.appservice_api.push(service, event) + @defer.inlineCallbacks + def _is_unknown_user(self, user_id): + user = UserID.from_string(user_id) + if not self.hs.is_mine(user): + # we don't know if they are unknown or not since it isn't one of our + # users. We can't poke ASes. + defer.returnValue(False) + return + + user_info = yield self.store.get_user_by_id(user_id) + defer.returnValue(len(user_info) == 0) + + + def _generate_hs_token(self): - return stringutils.random_string(18) + return stringutils.random_string(24) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 1daa314f20..f92dc1c89a 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -46,6 +46,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): ] self.mock_store.get_app_services = Mock(return_value=services) + self.mock_store.get_user_by_id = Mock(return_value=[]) event = Mock( sender="@someone:anywhere", -- cgit 1.4.1 From b932600653a4585968a7d177b7ea7cb2ca33642a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 11:47:11 +0000 Subject: Add unknown room alias check. Call it from directory_handler.get_association --- synapse/handlers/appservice.py | 97 +++++++++++++++++++++++------------------- synapse/handlers/directory.py | 7 +++ 2 files changed, 61 insertions(+), 43 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 39976b9629..2c6d4e2815 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -62,27 +62,6 @@ class ApplicationServicesHandler(object): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) - @defer.inlineCallbacks - def get_services_for_event(self, event, restrict_to=""): - """Retrieve a list of application services interested in this event. - - Args: - event(Event): The event to check. - restrict_to(str): The namespace to restrict regex tests to. - Returns: - list: A list of services interested in this - event based on the service regex. - """ - # We need to know the aliases associated with this event.room_id, if any - alias_list = yield self.store.get_aliases_for_room(event.room_id) - services = yield self.store.get_app_services() - interested_list = [ - s for s in services if ( - s.is_interested(event, restrict_to, alias_list) - ) - ] - defer.returnValue(interested_list) - @defer.inlineCallbacks def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. @@ -94,7 +73,7 @@ class ApplicationServicesHandler(object): event(Event): The event to push out to interested services. """ # Gather interested services - services = yield self.get_services_for_event(event) + services = yield self._get_services_for_event(event) if len(services) == 0: return # no services need notifying @@ -102,7 +81,7 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: - user_query_services = yield self.get_services_for_event( + user_query_services = yield self._get_services_for_event( event=event, restrict_to=ApplicationService.NS_USERS ) @@ -115,28 +94,62 @@ class ApplicationServicesHandler(object): # the user exists now,so don't query more ASes. break - # Do we know this room alias exists? If not, poke the room alias query - # API for all services which match that room alias regex. - unknown_room_alias = False # TODO - if unknown_room_alias: - alias = "something" # TODO - alias_query_services = yield self.get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_ALIASES - ) - for alias_service in alias_query_services: - # this needs to block XXX: Need to feed response back to caller - is_known_alias = yield self.appservice_api.query_alias( - alias_service, alias - ) - if is_known_alias: - # the alias exists now so don't query more ASes. - break - # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + + @defer.inlineCallbacks + def query_room_alias_exists(self, room_alias): + """Check if an application service knows this room alias exists. + + Args: + room_alias(str): The room alias to query. + Returns: + namedtuple: with keys "room_id" and "servers" or None if no + association can be found. + """ + alias_query_services = yield self._get_services_for_event( + event=None, + restrict_to=ApplicationService.NS_ALIASES, + alias_list=[room_alias] + ) + for alias_service in alias_query_services: + is_known_alias = yield self.appservice_api.query_alias( + alias_service, room_alias + ) + if is_known_alias: + # the alias exists now so don't query more ASes. + result = yield self.store.get_association_from_room_alias( + room_alias + ) + defer.returnValue(result) + + @defer.inlineCallbacks + def _get_services_for_event(self, event, restrict_to="", alias_list=None): + """Retrieve a list of application services interested in this event. + + Args: + event(Event): The event to check. Can be None if alias_list is not. + restrict_to(str): The namespace to restrict regex tests to. + alias_list: A list of aliases to get services for. If None, this + list is obtained from the database. + Returns: + list: A list of services interested in this + event based on the service regex. + """ + # We need to know the aliases associated with this event.room_id, if any + if not alias_list: + alias_list = yield self.store.get_aliases_for_room(event.room_id) + + services = yield self.store.get_app_services() + interested_list = [ + s for s in services if ( + s.is_interested(event, restrict_to, alias_list) + ) + ] + defer.returnValue(interested_list) + @defer.inlineCallbacks def _is_unknown_user(self, user_id): user = UserID.from_string(user_id) @@ -149,7 +162,5 @@ class ApplicationServicesHandler(object): user_info = yield self.store.get_user_by_id(user_id) defer.returnValue(len(user_info) == 0) - - def _generate_hs_token(self): return stringutils.random_string(24) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 58e9a91562..000bf5793c 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -84,6 +84,13 @@ class DirectoryHandler(BaseHandler): if result: room_id = result.room_id servers = result.servers + else: + # Query AS to see if it exists + as_handler = self.hs.get_handlers().appservice_handler + result = yield as_handler.query_room_alias_exists(room_alias) + if result: + room_id = result.room_id + servers = result.servers else: try: result = yield self.federation.make_query( -- cgit 1.4.1 From 26a041541baed887dc069c3667a86ddef81802bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Feb 2015 13:17:05 +0000 Subject: SYN-202: Log as WARN the 404 'Presence information not visible' errors instead of as ERROR since they were spamming the logs --- synapse/handlers/message.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6fbd2af4ab..3f51f38f18 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import RoomError +from synapse.api.errors import RoomError, SynapseError from synapse.streams.config import PaginationConfig from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator @@ -372,10 +372,17 @@ class MessageHandler(BaseHandler): as_event=True, ) presence.append(member_presence) - except Exception: - logger.exception( - "Failed to get member presence of %r", m.user_id - ) + except SynapseError as e: + if e.code == 404: + # FIXME: We are doing this as a warn since this gets hit a + # lot and spams the logs. Why is this happening? + logger.warn( + "Failed to get member presence of %r", m.user_id + ) + else: + logger.exception( + "Failed to get member presence of %r", m.user_id + ) time_now = self.clock.time_msec() -- cgit 1.4.1 From 51d63ac329a2613c0a7195e6183d70f789b7d823 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 13:19:46 +0000 Subject: Glue AS work to general event notifications. Add more exception handling when poking ASes. --- synapse/appservice/__init__.py | 2 +- synapse/appservice/api.py | 11 ++++++++++- synapse/handlers/appservice.py | 39 +++++++++++++++++++++++++-------------- synapse/notifier.py | 6 ++++++ 4 files changed, 42 insertions(+), 16 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f7baf578f0..d9ca856c8b 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -62,7 +62,7 @@ class ApplicationService(object): def _matches_regex(self, test_string, namespace_key): if not isinstance(test_string, basestring): - logger.warning( + logger.error( "Expected a string to test regex against, but got %s", test_string ) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 29bb35d61b..d96caf7f58 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -46,6 +46,9 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) return logger.warning("query_user to %s received %s", uri, e.code) + except Exception as ex: + logger.warning("query_user to %s threw exception %s", uri, ex) + defer.returnValue(False) @defer.inlineCallbacks def query_alias(self, service, alias): @@ -62,6 +65,10 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) return logger.warning("query_alias to %s received %s", uri, e.code) + except Exception as ex: + logger.warning("query_alias to %s threw exception %s", uri, ex) + defer.returnValue(False) + @defer.inlineCallbacks def push_bulk(self, service, events): @@ -82,7 +89,9 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(True) except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) - defer.returnValue(False) + except Exception as ex: + logger.warning("push_bulk to %s threw exception %s", uri, ex) + defer.returnValue(False) @defer.inlineCallbacks def push(self, service, event): diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2c6d4e2815..ef94215133 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -23,7 +23,6 @@ import synapse.util.stringutils as stringutils import logging - logger = logging.getLogger(__name__) @@ -58,6 +57,7 @@ class ApplicationServicesHandler(object): yield self.store.update_app_service(app_service) defer.returnValue(app_service) + @defer.inlineCallbacks def unregister(self, token): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) @@ -81,34 +81,45 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: - user_query_services = yield self._get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_USERS - ) - for user_service in user_query_services: - # this needs to block XXX: Need to feed response back to caller - is_known_user = yield self.appservice_api.query_user( - user_service, event.sender - ) - if is_known_user: - # the user exists now,so don't query more ASes. - break + yield self.query_user_exists(event) # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + @defer.inlineCallbacks + def query_user_exists(self, event): + """Check if an application services knows this event.sender exists. + + Args: + event: An event sent by the user to query + Returns: + True if this user exists. + """ + # TODO Would be nice for this to accept a user ID instead of an event. + user_query_services = yield self._get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_USERS + ) + for user_service in user_query_services: + is_known_user = yield self.appservice_api.query_user( + user_service, event.sender + ) + if is_known_user: + defer.returnValue(True) + defer.returnValue(False) @defer.inlineCallbacks def query_room_alias_exists(self, room_alias): """Check if an application service knows this room alias exists. Args: - room_alias(str): The room alias to query. + room_alias(RoomAlias): The room alias to query. Returns: namedtuple: with keys "room_id" and "servers" or None if no association can be found. """ + room_alias = room_alias.to_string() alias_query_services = yield self._get_services_for_event( event=None, restrict_to=ApplicationService.NS_ALIASES, diff --git a/synapse/notifier.py b/synapse/notifier.py index e3b6ead620..c7f75ab801 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -99,6 +99,12 @@ class Notifier(object): `extra_users` param. """ yield run_on_reactor() + + # poke any interested application service. + self.hs.get_handlers().appservice_handler.notify_interested_services( + event + ) + room_id = event.room_id room_source = self.event_sources.sources["room"] -- cgit 1.4.1 From 0613666d9c417005393636b935593c423f4417b9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 13:42:35 +0000 Subject: Serialize events before sending to ASes --- synapse/appservice/api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'synapse') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index d96caf7f58..9cce4e0973 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -16,6 +16,7 @@ from twisted.internet import defer from synapse.api.errors import CodeMessageException from synapse.http.client import SimpleHttpClient +from synapse.events.utils import serialize_event import logging import urllib @@ -30,6 +31,7 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) + self.clock = hs.get_clock() @defer.inlineCallbacks def query_user(self, service, user_id): @@ -72,6 +74,8 @@ class ApplicationServiceApi(SimpleHttpClient): @defer.inlineCallbacks def push_bulk(self, service, events): + events = self._serialize(events) + uri = service.url + ("/transactions/%s" % urllib.quote(str(0))) # TODO txn_ids response = None @@ -98,3 +102,9 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.push_bulk(service, [event]) defer.returnValue(response) + def _serialize(self, events): + time_now = self.clock.time_msec() + return [ + serialize_event(e, time_now, as_client_event=True) for e in events + ] + -- cgit 1.4.1 From e1515c3e91f2117adc3976b5e606728560ce9e96 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Feb 2015 13:43:28 +0000 Subject: Pass through list of room hosts from room alias query to federation so that it can retry against different room hosts --- synapse/federation/federation_client.py | 5 ++++- synapse/handlers/federation.py | 20 +++++++++++++------- synapse/handlers/room.py | 12 +++++------- 3 files changed, 22 insertions(+), 15 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index eb36ec040b..9923b3fc0d 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -264,7 +264,9 @@ class FederationClient(FederationBase): logger.debug("Got response to make_join: %s", pdu_dict) - defer.returnValue(self.event_from_pdu_json(pdu_dict)) + defer.returnValue( + (destination, self.event_from_pdu_json(pdu_dict)) + ) break except CodeMessageException: raise @@ -313,6 +315,7 @@ class FederationClient(FederationBase): defer.returnValue({ "state": signed_state, "auth_chain": signed_auth, + "origin": destination, }) except CodeMessageException: raise diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 04a4689483..aba266c2bc 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -273,7 +273,7 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks - def do_invite_join(self, target_host, room_id, joinee, content, snapshot): + def do_invite_join(self, target_hosts, room_id, joinee, content, snapshot): """ Attempts to join the `joinee` to the room `room_id` via the server `target_host`. @@ -287,8 +287,8 @@ class FederationHandler(BaseHandler): """ logger.debug("Joining %s to %s", joinee, room_id) - pdu = yield self.replication_layer.make_join( - [target_host], + origin, pdu = yield self.replication_layer.make_join( + target_hosts, room_id, joinee ) @@ -330,11 +330,17 @@ class FederationHandler(BaseHandler): new_event = builder.build() + # Try the host we successfully got a response to /make_join/ + # request first. + target_hosts.remove(origin) + target_hosts.insert(0, origin) + ret = yield self.replication_layer.send_join( - [target_host], + target_hosts, new_event ) + origin = ret["origin"] state = ret["state"] auth_chain = ret["auth_chain"] auth_chain.sort(key=lambda e: e.depth) @@ -371,7 +377,7 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } yield self._handle_new_event( - target_host, e, auth_events=auth + origin, e, auth_events=auth ) except: logger.exception( @@ -391,7 +397,7 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } yield self._handle_new_event( - target_host, e, auth_events=auth + origin, e, auth_events=auth ) except: logger.exception( @@ -406,7 +412,7 @@ class FederationHandler(BaseHandler): } yield self._handle_new_event( - target_host, + origin, new_event, state=state, current_state=state, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 23821d321f..0369b907a5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -389,8 +389,6 @@ class RoomMemberHandler(BaseHandler): if not hosts: raise SynapseError(404, "No known servers") - host = hosts[0] - # If event doesn't include a display name, add one. yield self.distributor.fire( "collect_presencelike_data", joinee, content @@ -407,12 +405,12 @@ class RoomMemberHandler(BaseHandler): }) event, context = yield self._create_new_client_event(builder) - yield self._do_join(event, context, room_host=host, do_auth=True) + yield self._do_join(event, context, room_hosts=hosts, do_auth=True) defer.returnValue({"room_id": room_id}) @defer.inlineCallbacks - def _do_join(self, event, context, room_host=None, do_auth=True): + def _do_join(self, event, context, room_hosts=None, do_auth=True): joinee = UserID.from_string(event.state_key) # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id @@ -441,7 +439,7 @@ class RoomMemberHandler(BaseHandler): if is_host_in_room: should_do_dance = False - elif room_host: # TODO: Shouldn't this be remote_room_host? + elif room_hosts: # TODO: Shouldn't this be remote_room_host? should_do_dance = True else: # TODO(markjh): get prev_state from snapshot @@ -453,7 +451,7 @@ class RoomMemberHandler(BaseHandler): inviter = UserID.from_string(prev_state.user_id) should_do_dance = not self.hs.is_mine(inviter) - room_host = inviter.domain + room_hosts = [inviter.domain] else: # return the same error as join_room_alias does raise SynapseError(404, "No known servers") @@ -461,7 +459,7 @@ class RoomMemberHandler(BaseHandler): if should_do_dance: handler = self.hs.get_handlers().federation_handler yield handler.do_invite_join( - room_host, + room_hosts, room_id, event.user_id, event.get_dict()["content"], # FIXME To get a non-frozen dict -- cgit 1.4.1 From e9c85a4d5ab332021f93634182ad8ed93bd0091c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Feb 2015 13:50:15 +0000 Subject: Connection errors in twisted aren't RuntimeErrors --- synapse/federation/federation_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 9923b3fc0d..70c9a6f46b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -270,7 +270,7 @@ class FederationClient(FederationBase): break except CodeMessageException: raise - except RuntimeError as e: + except Exception as e: logger.warn( "Failed to make_join via %s: %s", destination, e.message @@ -319,7 +319,7 @@ class FederationClient(FederationBase): }) except CodeMessageException: raise - except RuntimeError as e: + except Exception as e: logger.warn( "Failed to send_join via %s: %s", destination, e.message -- cgit 1.4.1 From c71456117dc70dfe0bfa15e3f655a3ac1dfc66ee Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 14:17:08 +0000 Subject: Fix user query checks. HS>AS pushing now works. --- synapse/appservice/__init__.py | 21 ++++++++++++-------- synapse/handlers/appservice.py | 44 +++++++++++++++++++++++++++++------------- 2 files changed, 44 insertions(+), 21 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index d9ca856c8b..46d46a5a48 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -75,27 +75,23 @@ class ApplicationService(object): def _matches_user(self, event): if (hasattr(event, "sender") and - self._matches_regex( - event.sender, ApplicationService.NS_USERS)): + self.is_interested_in_user(event.sender)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member and hasattr(event, "state_key") - and self._matches_regex( - event.state_key, ApplicationService.NS_USERS)): + and self.is_interested_in_user(event.state_key)): return True return False def _matches_room_id(self, event): if hasattr(event, "room_id"): - return self._matches_regex( - event.room_id, ApplicationService.NS_ROOMS - ) + return self.is_interested_in_room(event.room_id) return False def _matches_aliases(self, event, alias_list): for alias in alias_list: - if self._matches_regex(alias, ApplicationService.NS_ALIASES): + if self.is_interested_in_alias(alias): return True return False @@ -128,5 +124,14 @@ class ApplicationService(object): elif restrict_to == ApplicationService.NS_USERS: return self._matches_user(event) + def is_interested_in_user(self, user_id): + return self._matches_regex(user_id, ApplicationService.NS_USERS) + + def is_interested_in_alias(self, alias): + return self._matches_regex(alias, ApplicationService.NS_ALIASES) + + def is_interested_in_room(self, room_id): + return self._matches_regex(room_id, ApplicationService.NS_ROOMS) + def __str__(self): return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index ef94215133..8d0cdd528c 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,6 +15,7 @@ from twisted.internet import defer +from synapse.api.constants import EventTypes from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi @@ -78,32 +79,31 @@ class ApplicationServicesHandler(object): return # no services need notifying # Do we know this user exists? If not, poke the user query API for - # all services which match that user regex. - unknown_user = yield self._is_unknown_user(event.sender) - if unknown_user: - yield self.query_user_exists(event) + # all services which match that user regex. This needs to block as these + # user queries need to be made BEFORE pushing the event. + yield self._check_user_exists(event.sender) + if event.type == EventTypes.Member: + yield self._check_user_exists(event.state_key) # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) @defer.inlineCallbacks - def query_user_exists(self, event): - """Check if an application services knows this event.sender exists. + def query_user_exists(self, user_id): + """Check if any application service knows this user_id exists. Args: - event: An event sent by the user to query + user_id(str): The user to query if they exist on any AS. Returns: - True if this user exists. + True if this user exists on at least one application service. """ - # TODO Would be nice for this to accept a user ID instead of an event. - user_query_services = yield self._get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_USERS + user_query_services = yield self._get_services_for_user( + user_id=user_id ) for user_service in user_query_services: is_known_user = yield self.appservice_api.query_user( - user_service, event.sender + user_service, user_id ) if is_known_user: defer.returnValue(True) @@ -161,6 +161,16 @@ class ApplicationServicesHandler(object): ] defer.returnValue(interested_list) + @defer.inlineCallbacks + def _get_services_for_user(self, user_id): + services = yield self.store.get_app_services() + interested_list = [ + s for s in services if ( + s.is_interested_in_user(user_id) + ) + ] + defer.returnValue(interested_list) + @defer.inlineCallbacks def _is_unknown_user(self, user_id): user = UserID.from_string(user_id) @@ -173,5 +183,13 @@ class ApplicationServicesHandler(object): user_info = yield self.store.get_user_by_id(user_id) defer.returnValue(len(user_info) == 0) + @defer.inlineCallbacks + def _check_user_exists(self, user_id): + unknown_user = yield self._is_unknown_user(user_id) + if unknown_user: + exists = yield self.query_user_exists(user_id) + defer.returnValue(exists) + defer.returnValue(True) + def _generate_hs_token(self): return stringutils.random_string(24) -- cgit 1.4.1 From f90782a6589ee567d6dc9ead19451cc8a0aabdd3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 5 Feb 2015 09:35:21 +0000 Subject: namespace rule IDs to be unique within their scope and rule type. --- synapse/rest/client/v1/push_rule.py | 252 ++++++++++++++++++------------------ 1 file changed, 129 insertions(+), 123 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 5582f33c8e..eaef55cc1e 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -38,100 +38,9 @@ class PushRuleRestServlet(ClientV1RestServlet): SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( "Unrecognised request: You probably wanted a trailing slash") - def rule_spec_from_path(self, path): - if len(path) < 2: - raise UnrecognizedRequestError() - if path[0] != 'pushrules': - raise UnrecognizedRequestError() - - scope = path[1] - path = path[2:] - if scope not in ['global', 'device']: - raise UnrecognizedRequestError() - - device = None - if scope == 'device': - if len(path) == 0: - raise UnrecognizedRequestError() - device = path[0] - path = path[1:] - - if len(path) == 0: - raise UnrecognizedRequestError() - - template = path[0] - path = path[1:] - - if len(path) == 0: - raise UnrecognizedRequestError() - - rule_id = path[0] - - spec = { - 'scope': scope, - 'template': template, - 'rule_id': rule_id - } - if device: - spec['profile_tag'] = device - return spec - - def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj, device=None): - if rule_template in ['override', 'underride']: - if 'conditions' not in req_obj: - raise InvalidRuleException("Missing 'conditions'") - conditions = req_obj['conditions'] - for c in conditions: - if 'kind' not in c: - raise InvalidRuleException("Condition without 'kind'") - elif rule_template == 'room': - conditions = [{ - 'kind': 'event_match', - 'key': 'room_id', - 'pattern': rule_id - }] - elif rule_template == 'sender': - conditions = [{ - 'kind': 'event_match', - 'key': 'user_id', - 'pattern': rule_id - }] - elif rule_template == 'content': - if 'pattern' not in req_obj: - raise InvalidRuleException("Content rule missing 'pattern'") - pat = req_obj['pattern'] - - conditions = [{ - 'kind': 'event_match', - 'key': 'content.body', - 'pattern': pat - }] - else: - raise InvalidRuleException("Unknown rule template: %s" % (rule_template,)) - - if device: - conditions.append({ - 'kind': 'device', - 'profile_tag': device - }) - - if 'actions' not in req_obj: - raise InvalidRuleException("No actions found") - actions = req_obj['actions'] - - for a in actions: - if a in ['notify', 'dont_notify', 'coalesce']: - pass - elif isinstance(a, dict) and 'set_sound' in a: - pass - else: - raise InvalidRuleException("Unrecognised action") - - return conditions, actions - @defer.inlineCallbacks def on_PUT(self, request): - spec = self.rule_spec_from_path(request.postpath) + spec = _rule_spec_from_path(request.postpath) try: priority_class = _priority_class_from_spec(spec) except InvalidRuleException as e: @@ -142,10 +51,13 @@ class PushRuleRestServlet(ClientV1RestServlet): if spec['template'] == 'default': raise SynapseError(403, "The default rules are immutable.") + if not spec['rule_id'].isalnum(): + raise SynapseError(400, "rule_id may only contain alphanumeric characters") + content = _parse_json(request) try: - (conditions, actions) = self.rule_tuple_from_request_object( + (conditions, actions) = _rule_tuple_from_request_object( spec['template'], spec['rule_id'], content, @@ -164,7 +76,7 @@ class PushRuleRestServlet(ClientV1RestServlet): try: yield self.hs.get_datastore().add_push_rule( user_name=user.to_string(), - rule_id=spec['rule_id'], + rule_id=_namespaced_rule_id_from_spec(spec), priority_class=priority_class, conditions=conditions, actions=actions, @@ -180,7 +92,7 @@ class PushRuleRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request): - spec = self.rule_spec_from_path(request.postpath) + spec = _rule_spec_from_path(request.postpath) try: priority_class = _priority_class_from_spec(spec) except InvalidRuleException as e: @@ -188,32 +100,18 @@ class PushRuleRestServlet(ClientV1RestServlet): user, _ = yield self.auth.get_user_by_req(request) - if 'profile_tag' in spec: - rules = yield self.hs.get_datastore().get_push_rules_for_user_name( - user.to_string() - ) + namespaced_rule_id = _namespaced_rule_id_from_spec(spec) - for r in rules: - conditions = json.loads(r['conditions']) - pt = _profile_tag_from_conditions(conditions) - if pt == spec['profile_tag'] and r['priority_class'] == priority_class: - yield self.hs.get_datastore().delete_push_rule( - user.to_string(), spec['rule_id'] - ) - defer.returnValue((200, {})) - raise NotFoundError() - else: - try: - yield self.hs.get_datastore().delete_push_rule( - user.to_string(), spec['rule_id'], - priority_class=priority_class - ) - defer.returnValue((200, {})) - except StoreError as e: - if e.code == 404: - raise NotFoundError() - else: - raise + try: + yield self.hs.get_datastore().delete_push_rule( + user.to_string(), namespaced_rule_id + ) + defer.returnValue((200, {})) + except StoreError as e: + if e.code == 404: + raise NotFoundError() + else: + raise @defer.inlineCallbacks def on_GET(self, request): @@ -298,6 +196,99 @@ class PushRuleRestServlet(ClientV1RestServlet): return 200, {} +def _rule_spec_from_path(path): + if len(path) < 2: + raise UnrecognizedRequestError() + if path[0] != 'pushrules': + raise UnrecognizedRequestError() + + scope = path[1] + path = path[2:] + if scope not in ['global', 'device']: + raise UnrecognizedRequestError() + + device = None + if scope == 'device': + if len(path) == 0: + raise UnrecognizedRequestError() + device = path[0] + path = path[1:] + + if len(path) == 0: + raise UnrecognizedRequestError() + + template = path[0] + path = path[1:] + + if len(path) == 0: + raise UnrecognizedRequestError() + + rule_id = path[0] + + spec = { + 'scope': scope, + 'template': template, + 'rule_id': rule_id + } + if device: + spec['profile_tag'] = device + return spec + + +def _rule_tuple_from_request_object(rule_template, rule_id, req_obj, device=None): + if rule_template in ['override', 'underride']: + if 'conditions' not in req_obj: + raise InvalidRuleException("Missing 'conditions'") + conditions = req_obj['conditions'] + for c in conditions: + if 'kind' not in c: + raise InvalidRuleException("Condition without 'kind'") + elif rule_template == 'room': + conditions = [{ + 'kind': 'event_match', + 'key': 'room_id', + 'pattern': rule_id + }] + elif rule_template == 'sender': + conditions = [{ + 'kind': 'event_match', + 'key': 'user_id', + 'pattern': rule_id + }] + elif rule_template == 'content': + if 'pattern' not in req_obj: + raise InvalidRuleException("Content rule missing 'pattern'") + pat = req_obj['pattern'] + + conditions = [{ + 'kind': 'event_match', + 'key': 'content.body', + 'pattern': pat + }] + else: + raise InvalidRuleException("Unknown rule template: %s" % (rule_template,)) + + if device: + conditions.append({ + 'kind': 'device', + 'profile_tag': device + }) + + if 'actions' not in req_obj: + raise InvalidRuleException("No actions found") + actions = req_obj['actions'] + + for a in actions: + if a in ['notify', 'dont_notify', 'coalesce']: + pass + elif isinstance(a, dict) and 'set_sound' in a: + pass + else: + raise InvalidRuleException("Unrecognised action") + + return conditions, actions + + def _add_empty_priority_class_arrays(d): for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): d[pc] = [] @@ -361,20 +352,24 @@ def _priority_class_to_template_name(pc): def _rule_to_template(rule): + unscoped_rule_id = _rule_id_from_namespaced(rule['rule_id']) + template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['default']: return {k: rule[k] for k in ["conditions", "actions"]} elif template_name in ['override', 'underride']: - return {k: rule[k] for k in ["rule_id", "conditions", "actions"]} + ret = {k: rule[k] for k in ["conditions", "actions"]} + ret['rule_id'] = unscoped_rule_id + return ret elif template_name in ["sender", "room"]: - return {k: rule[k] for k in ["rule_id", "actions"]} + return {'rule_id': unscoped_rule_id, 'actions': rule['actions']} elif template_name == 'content': if len(rule["conditions"]) != 1: return None thecond = rule["conditions"][0] if "pattern" not in thecond: return None - ret = {k: rule[k] for k in ["rule_id", "actions"]} + ret = {'rule_id': unscoped_rule_id, 'actions': rule['actions']} ret["pattern"] = thecond["pattern"] return ret @@ -386,6 +381,17 @@ def _strip_device_condition(rule): return rule +def _namespaced_rule_id_from_spec(spec): + if spec['scope'] == 'global': + scope = 'global' + else: + scope = 'device.%s' % (spec['profile_tag']) + return "%s.%s.%s" % (scope, spec['template'], spec['rule_id']) + + +def _rule_id_from_namespaced(in_rule_id, spec): + return in_rule_id.split('.')[-1] + class InvalidRuleException(Exception): pass -- cgit 1.4.1 From 2df41aa1386545f4237c0141c19db1fef85e7161 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 5 Feb 2015 14:46:06 +0000 Subject: Server default rules now of all kinds rather than all being at lowest prio. --- synapse/push/__init__.py | 8 ++--- synapse/push/baserules.py | 62 +++++++++++++++++++++++++++----- synapse/push/rulekinds.py | 8 +++++ synapse/rest/client/v1/push_rule.py | 71 +++++++++++++++++-------------------- 4 files changed, 98 insertions(+), 51 deletions(-) create mode 100644 synapse/push/rulekinds.py (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8c6f0a6571..7293715293 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -77,15 +77,15 @@ class Pusher(object): if ev['state_key'] != self.user_name: defer.returnValue(['dont_notify']) - rules = yield self.store.get_push_rules_for_user_name(self.user_name) + rawrules = yield self.store.get_push_rules_for_user_name(self.user_name) - for r in rules: + for r in rawrules: r['conditions'] = json.loads(r['conditions']) r['actions'] = json.loads(r['actions']) - user_name_localpart = UserID.from_string(self.user_name).localpart + user = UserID.from_string(self.user_name) - rules.extend(baserules.make_base_rules(user_name_localpart)) + rules = baserules.list_with_base_rules(rawrules, user) # get *our* member event for display name matching member_events_for_room = yield self.store.get_current_state( diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 376d1d4d33..191909ad4d 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -1,20 +1,68 @@ -def make_base_rules(user_name): - rules = [ +from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP + +def list_with_base_rules(rawrules, user_name): + ruleslist = [] + + # shove the server default rules for each kind onto the end of each + current_prio_class = 1 + for r in rawrules: + if r['priority_class'] > current_prio_class: + while current_prio_class < r['priority_class']: + ruleslist.extend(make_base_rules( + user_name, + PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) + ) + current_prio_class += 1 + + ruleslist.append(r) + + while current_prio_class <= PRIORITY_CLASS_INVERSE_MAP.keys()[-1]: + ruleslist.extend(make_base_rules( + user_name, + PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) + ) + current_prio_class += 1 + + return ruleslist + + +def make_base_rules(user, kind): + rules = [] + + if kind == 'override': + rules = make_base_override_rules() + elif kind == 'content': + rules = make_base_content_rules(user) + + for r in rules: + r['priority_class'] = PRIORITY_CLASS_MAP[kind] + + return rules + + +def make_base_content_rules(user): + return [ { 'conditions': [ { 'kind': 'event_match', 'key': 'content.body', - 'pattern': '*%s*' % (user_name,), # Matrix ID match + 'pattern': user.localpart, # Matrix ID match } ], 'actions': [ 'notify', { - 'set_sound': 'default' + 'set_tweak': 'sound', + 'value': 'default', } ] }, + ] + + +def make_base_override_rules(): + return [ { 'conditions': [ { @@ -24,7 +72,8 @@ def make_base_rules(user_name): 'actions': [ 'notify', { - 'set_sound': 'default' + 'set_tweak': 'sound', + 'value': 'default' } ] }, @@ -44,6 +93,3 @@ def make_base_rules(user_name): ] } ] - for r in rules: - r['priority_class'] = 0 - return rules \ No newline at end of file diff --git a/synapse/push/rulekinds.py b/synapse/push/rulekinds.py new file mode 100644 index 0000000000..763bdee58e --- /dev/null +++ b/synapse/push/rulekinds.py @@ -0,0 +1,8 @@ +PRIORITY_CLASS_MAP = { + 'underride': 1, + 'sender': 2, + 'room': 3, + 'content': 4, + 'override': 5, + } +PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index eaef55cc1e..7ab167ce03 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -20,21 +20,13 @@ from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, No from .base import ClientV1RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException import synapse.push.baserules as baserules +from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP import json class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") - PRIORITY_CLASS_MAP = { - 'default': 0, - 'underride': 1, - 'sender': 2, - 'room': 3, - 'content': 4, - 'override': 5, - } - PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( "Unrecognised request: You probably wanted a trailing slash") @@ -48,11 +40,8 @@ class PushRuleRestServlet(ClientV1RestServlet): user, _ = yield self.auth.get_user_by_req(request) - if spec['template'] == 'default': - raise SynapseError(403, "The default rules are immutable.") - - if not spec['rule_id'].isalnum(): - raise SynapseError(400, "rule_id may only contain alphanumeric characters") + if '/' in spec['rule_id'] or '\\' in spec['rule_id']: + raise SynapseError(400, "rule_id may not contain slashes") content = _parse_json(request) @@ -121,21 +110,23 @@ class PushRuleRestServlet(ClientV1RestServlet): # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + for r in rawrules: r["conditions"] = json.loads(r["conditions"]) r["actions"] = json.loads(r["actions"]) - rawrules.extend(baserules.make_base_rules(user.to_string())) + + ruleslist = baserules.list_with_base_rules(rawrules, user) rules = {'global': {}, 'device': {}} rules['global'] = _add_empty_priority_class_arrays(rules['global']) - for r in rawrules: + for r in ruleslist: rulearray = None template_name = _priority_class_to_template_name(r['priority_class']) - if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: + if r['priority_class'] > PRIORITY_CLASS_MAP['override']: # per-device rule profile_tag = _profile_tag_from_conditions(r["conditions"]) r = _strip_device_condition(r) @@ -290,7 +281,7 @@ def _rule_tuple_from_request_object(rule_template, rule_id, req_obj, device=None def _add_empty_priority_class_arrays(d): - for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): + for pc in PRIORITY_CLASS_MAP.keys(): d[pc] = [] return d @@ -332,46 +323,48 @@ def _filter_ruleset_with_path(ruleset, path): def _priority_class_from_spec(spec): - if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): + if spec['template'] not in PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) - pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] + pc = PRIORITY_CLASS_MAP[spec['template']] if spec['scope'] == 'device': - pc += len(PushRuleRestServlet.PRIORITY_CLASS_MAP) + pc += len(PRIORITY_CLASS_MAP) return pc def _priority_class_to_template_name(pc): - if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: + if pc > PRIORITY_CLASS_MAP['override']: # per-device prio_class_index = pc - len(PushRuleRestServlet.PRIORITY_CLASS_MAP) - return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[prio_class_index] + return PRIORITY_CLASS_INVERSE_MAP[prio_class_index] else: - return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + return PRIORITY_CLASS_INVERSE_MAP[pc] def _rule_to_template(rule): - unscoped_rule_id = _rule_id_from_namespaced(rule['rule_id']) + unscoped_rule_id = None + if 'rule_id' in rule: + _rule_id_from_namespaced(rule['rule_id']) template_name = _priority_class_to_template_name(rule['priority_class']) - if template_name in ['default']: - return {k: rule[k] for k in ["conditions", "actions"]} - elif template_name in ['override', 'underride']: - ret = {k: rule[k] for k in ["conditions", "actions"]} - ret['rule_id'] = unscoped_rule_id - return ret + if template_name in ['override', 'underride']: + templaterule = {k: rule[k] for k in ["conditions", "actions"]} elif template_name in ["sender", "room"]: - return {'rule_id': unscoped_rule_id, 'actions': rule['actions']} + templaterule = {'actions': rule['actions']} + unscoped_rule_id = rule['conditions'][0]['pattern'] elif template_name == 'content': if len(rule["conditions"]) != 1: return None thecond = rule["conditions"][0] if "pattern" not in thecond: return None - ret = {'rule_id': unscoped_rule_id, 'actions': rule['actions']} - ret["pattern"] = thecond["pattern"] - return ret + templaterule = {'actions': rule['actions']} + templaterule["pattern"] = thecond["pattern"] + + if unscoped_rule_id: + templaterule['rule_id'] = unscoped_rule_id + return templaterule def _strip_device_condition(rule): @@ -385,12 +378,12 @@ def _namespaced_rule_id_from_spec(spec): if spec['scope'] == 'global': scope = 'global' else: - scope = 'device.%s' % (spec['profile_tag']) - return "%s.%s.%s" % (scope, spec['template'], spec['rule_id']) + scope = 'device/%s' % (spec['profile_tag']) + return "%s/%s/%s" % (scope, spec['template'], spec['rule_id']) -def _rule_id_from_namespaced(in_rule_id, spec): - return in_rule_id.split('.')[-1] +def _rule_id_from_namespaced(in_rule_id): + return in_rule_id.split('/')[-1] class InvalidRuleException(Exception): pass -- cgit 1.4.1 From c163357f387788afa8fc1d4160e806eccbc5220a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 15:00:33 +0000 Subject: Add CS extension for masquerading as users within the namespaces specified by the AS. --- synapse/api/auth.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 7105ee21dc..86f963c190 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -299,6 +299,29 @@ class Auth(object): # Can optionally look elsewhere in the request (e.g. headers) try: access_token = request.args["access_token"][0] + + # Check for application service tokens with a user_id override + try: + masquerade_user_id = request.args["user_id"][0] + app_service = yield self.store.get_app_service_by_token( + access_token + ) + if not app_service: + raise AuthError( + 403, "Invalid application service access token" + ) + if not app_service.is_interested_in_user(masquerade_user_id): + raise AuthError( + 403, + "Application service cannot masquerade as this user." + ) + defer.returnValue( + (UserID.from_string(masquerade_user_id), ClientInfo("", "")) + ) + return + except KeyError: + pass # normal users won't have this query parameter set + user_info = yield self.get_user_by_token(access_token) user = user_info["user"] device_id = user_info["device_id"] -- cgit 1.4.1 From aaf50bf6f3d6adee92fa4d5cb55dbf3c5a13dbe3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 5 Feb 2015 15:11:38 +0000 Subject: Give server default rules the 'default' attribute and fix various brokenness. --- synapse/push/baserules.py | 1 + synapse/rest/client/v1/push_rule.py | 4 +++- synapse/storage/push_rule.py | 7 +++++-- 3 files changed, 9 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 191909ad4d..8d4b806da6 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -36,6 +36,7 @@ def make_base_rules(user, kind): for r in rules: r['priority_class'] = PRIORITY_CLASS_MAP[kind] + r['default'] = True return rules diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 7ab167ce03..80f116b1ed 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -345,7 +345,7 @@ def _priority_class_to_template_name(pc): def _rule_to_template(rule): unscoped_rule_id = None if 'rule_id' in rule: - _rule_id_from_namespaced(rule['rule_id']) + unscoped_rule_id = _rule_id_from_namespaced(rule['rule_id']) template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: @@ -364,6 +364,8 @@ def _rule_to_template(rule): if unscoped_rule_id: templaterule['rule_id'] = unscoped_rule_id + if 'default' in rule: + templaterule['default'] = rule['default'] return templaterule diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 27502d2399..30e23445d9 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -176,7 +176,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) @defer.inlineCallbacks - def delete_push_rule(self, user_name, rule_id, **kwargs): + def delete_push_rule(self, user_name, rule_id): """ Delete a push rule. Args specify the row to be deleted and can be any of the columns in the push_rule table, but below are the @@ -186,7 +186,10 @@ class PushRuleStore(SQLBaseStore): user_name (str): The matrix ID of the push rule owner rule_id (str): The rule_id of the rule to be deleted """ - yield self._simple_delete_one(PushRuleTable.table_name, kwargs) + yield self._simple_delete_one( + PushRuleTable.table_name, + {'user_name': user_name, 'rule_id': rule_id} + ) class RuleNotFoundException(Exception): -- cgit 1.4.1 From 5b99b471b2ba9a9c9da801e9a9d6e1801f02068c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 15:12:36 +0000 Subject: Fix unit tests. --- synapse/api/auth.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 86f963c190..4f116184c9 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -302,6 +302,12 @@ class Auth(object): # Check for application service tokens with a user_id override try: + if "user_id" not in request.args: + # This has to be done like this rather than relying on it + # natively throwing because tests use a Mock for the request + # object which doesn't throw :/ + raise KeyError + masquerade_user_id = request.args["user_id"][0] app_service = yield self.store.get_app_service_by_token( access_token -- cgit 1.4.1 From a93fa42bcef97b1e5f938a3a96f5afcbaeaef376 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 5 Feb 2015 15:45:16 +0000 Subject: priority class now dealt with in namespaced rule_id --- synapse/rest/client/v1/push_rule.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 80f116b1ed..d43ade39dd 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -82,10 +82,6 @@ class PushRuleRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request): spec = _rule_spec_from_path(request.postpath) - try: - priority_class = _priority_class_from_spec(spec) - except InvalidRuleException as e: - raise SynapseError(400, e.message) user, _ = yield self.auth.get_user_by_req(request) -- cgit 1.4.1 From e9484d6a9514d5c9aea3839bcc3fb25c9cb88cb0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 16:29:56 +0000 Subject: Prevent aliases in AS namespaces being created/deleted by users. Check with ASes when queried for room aliases via federation. --- synapse/handlers/directory.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 24ea3573d3..842f075fe0 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -49,6 +49,12 @@ class DirectoryHandler(BaseHandler): # TODO(erikj): Check if there is a current association. + is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) + if is_claimed: + raise SynapseError( + 400, "This alias is reserved by an application service." + ) + if not servers: servers = yield self.store.get_joined_hosts_for_room(room_id) @@ -68,6 +74,12 @@ class DirectoryHandler(BaseHandler): if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") + is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) + if is_claimed: + raise SynapseError( + 400, "This alias is reserved by an application service." + ) + room_id = yield self.store.delete_room_alias(room_alias) if room_id: @@ -77,20 +89,13 @@ class DirectoryHandler(BaseHandler): def get_association(self, room_alias): room_id = None if self.hs.is_mine(room_alias): - result = yield self.store.get_association_from_room_alias( + result = yield self.get_association_from_room_alias( room_alias ) if result: room_id = result.room_id servers = result.servers - else: - # Query AS to see if it exists - as_handler = self.hs.get_handlers().appservice_handler - result = yield as_handler.query_room_alias_exists(room_alias) - if result: - room_id = result.room_id - servers = result.servers else: try: result = yield self.federation.make_query( @@ -145,7 +150,7 @@ class DirectoryHandler(BaseHandler): 400, "Room Alias is not hosted on this Home Server" ) - result = yield self.store.get_association_from_room_alias( + result = yield self.get_association_from_room_alias( room_alias ) @@ -173,3 +178,22 @@ class DirectoryHandler(BaseHandler): "sender": user_id, "content": {"aliases": aliases}, }, ratelimit=False) + + @defer.inlineCallbacks + def get_association_from_room_alias(self, room_alias): + result = yield self.store.get_association_from_room_alias( + room_alias + ) + if not result: + # Query AS to see if it exists + as_handler = self.hs.get_handlers().appservice_handler + result = yield as_handler.query_room_alias_exists(room_alias) + defer.returnValue(result) + + @defer.inlineCallbacks + def is_alias_exclusive_to_appservices(self, alias): + services = yield self.store.get_app_services() + interested_services = [ + s for s in services if s.is_interested_in_alias(alias.to_string()) + ] + defer.returnValue(len(interested_services) > 0) -- cgit 1.4.1 From cab4c730885dcb5c95a50d425d6b6f655154a173 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 16:46:56 +0000 Subject: Prevent user IDs in AS namespaces being created/deleted by humans. --- synapse/handlers/register.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'synapse') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 08cd5fd720..b6e19d498c 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -65,6 +65,8 @@ class RegistrationHandler(BaseHandler): user = UserID(localpart, self.hs.hostname) user_id = user.to_string() + yield self.check_user_id_is_valid(user_id) + token = self._generate_token(user_id) yield self.store.register( user_id=user_id, @@ -83,6 +85,7 @@ class RegistrationHandler(BaseHandler): localpart = self._generate_user_id() user = UserID(localpart, self.hs.hostname) user_id = user.to_string() + yield self.check_user_id_is_valid(user_id) token = self._generate_token(user_id) yield self.store.register( @@ -148,6 +151,19 @@ class RegistrationHandler(BaseHandler): # XXX: This should be a deferred list, shouldn't it? yield self._bind_threepid(c, user_id) + @defer.inlineCallbacks + def check_user_id_is_valid(self, user_id): + # valid user IDs must not clash with any user ID namespaces claimed by + # application services. + services = yield self.store.get_app_services() + interested_services = [ + s for s in services if s.is_interested_in_user(user_id) + ] + if len(interested_services) > 0: + raise SynapseError( + 400, "This user ID is reserved by an application service." + ) + def _generate_token(self, user_id): # urlsafe variant uses _ and - so use . as the separator and replace # all =s with .s so http clients don't quote =s when it is used as -- cgit 1.4.1 From a3c6010718c2749bd446bb63f3cf03bae09b0d20 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 16:48:57 +0000 Subject: Add delta sql file. --- synapse/storage/schema/delta/v14.sql | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 synapse/storage/schema/delta/v14.sql (limited to 'synapse') diff --git a/synapse/storage/schema/delta/v14.sql b/synapse/storage/schema/delta/v14.sql new file mode 100644 index 0000000000..03b5a10c8a --- /dev/null +++ b/synapse/storage/schema/delta/v14.sql @@ -0,0 +1,33 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT, + token TEXT, + hs_token TEXT, + UNIQUE(token) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id INTEGER PRIMARY KEY AUTOINCREMENT, + as_id INTEGER NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex TEXT, + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + + + -- cgit 1.4.1 From 11e6b3d18b5ae95857e01884960a01b9ea4d307d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 17:04:59 +0000 Subject: Dependency inject ApplicationServiceApi when creating ApplicationServicesHandler. --- synapse/handlers/__init__.py | 5 ++++- synapse/handlers/appservice.py | 5 ++--- tests/handlers/test_appservice.py | 10 +++------- 3 files changed, 9 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index b31518bf62..8d345bf936 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.appservice.api import ApplicationServiceApi from .register import RegistrationHandler from .room import ( RoomCreationHandler, RoomMemberHandler, RoomListHandler @@ -53,5 +54,7 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) - self.appservice_handler = ApplicationServicesHandler(hs) + self.appservice_handler = ApplicationServicesHandler( + hs, ApplicationServiceApi(hs) + ) self.sync_handler = SyncHandler(hs) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8d0cdd528c..fa810b9a98 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -18,7 +18,6 @@ from twisted.internet import defer from synapse.api.constants import EventTypes from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService -from synapse.appservice.api import ApplicationServiceApi from synapse.types import UserID import synapse.util.stringutils as stringutils @@ -32,10 +31,10 @@ logger = logging.getLogger(__name__) # easier. class ApplicationServicesHandler(object): - def __init__(self, hs): + def __init__(self, hs, appservice_api): self.store = hs.get_datastore() self.hs = hs - self.appservice_api = ApplicationServiceApi(hs) + self.appservice_api = appservice_api @defer.inlineCallbacks def register(self, app_service): diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index e16e511587..a2c541317c 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -29,13 +29,9 @@ class AppServiceHandlerTestCase(unittest.TestCase): self.mock_as_api = Mock() hs = Mock() hs.get_datastore = Mock(return_value=self.mock_store) - self.handler = ApplicationServicesHandler(hs) # thing being tested - - # FIXME Would be nice to DI this rather than monkey patch:( - if not hasattr(self.handler, "appservice_api"): - # someone probably updated the handler but not the tests. Fail fast. - raise Exception("Test expected handler.appservice_api to exist.") - self.handler.appservice_api = self.mock_as_api + self.handler = ApplicationServicesHandler( + hs, self.mock_as_api + ) @defer.inlineCallbacks def test_notify_interested_services(self): -- cgit 1.4.1 From 0227618d3c3bea6c85a922d5605f526719573121 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 17:29:27 +0000 Subject: Add m.login.application_service registration procedure. This allows known application services to register any user ID under their own user namespace(s). --- synapse/api/constants.py | 1 + synapse/handlers/register.py | 20 ++++++++++++++++++++ synapse/rest/client/v1/register.py | 24 +++++++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 0d3fc629af..420f963d91 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -59,6 +59,7 @@ class LoginType(object): EMAIL_URL = u"m.login.email.url" EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" + APPLICATION_SERVICE = u"m.login.application_service" class EventTypes(object): diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index b6e19d498c..60821edb05 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -105,6 +105,26 @@ class RegistrationHandler(BaseHandler): defer.returnValue((user_id, token)) + @defer.inlineCallbacks + def appservice_register(self, user_localpart, as_token): + user = UserID(user_localpart, self.hs.hostname) + user_id = user.to_string() + service = yield self.store.get_app_service_by_token(as_token) + if not service: + raise SynapseError(403, "Invalid application service token.") + if not service.is_interested_in_user(user_id): + raise SynapseError( + 400, "Invalid user localpart for this application service." + ) + token = self._generate_token(user_id) + yield self.store.register( + user_id=user_id, + token=token, + password_hash="" + ) + self.distributor.fire("registered_user", user) + defer.returnValue((user_id, token)) + @defer.inlineCallbacks def check_recaptcha(self, ip, private_key, challenge, response): """Checks a recaptcha is correct.""" diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index c0423c2d45..1ab32b53ea 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -110,7 +110,8 @@ class RegisterRestServlet(ClientV1RestServlet): stages = { LoginType.RECAPTCHA: self._do_recaptcha, LoginType.PASSWORD: self._do_password, - LoginType.EMAIL_IDENTITY: self._do_email_identity + LoginType.EMAIL_IDENTITY: self._do_email_identity, + LoginType.APPLICATION_SERVICE: self._do_app_service } session_info = self._get_session_info(request, session) @@ -276,6 +277,27 @@ class RegisterRestServlet(ClientV1RestServlet): self._remove_session(session) defer.returnValue(result) + @defer.inlineCallbacks + def _do_app_service(self, request, register_json, session): + if "access_token" not in request.args: + raise SynapseError(400, "Expected application service token.") + if "user" not in register_json: + raise SynapseError(400, "Expected 'user' key.") + + as_token = request.args["access_token"][0] + user_localpart = register_json["user"].encode("utf-8") + + handler = self.handlers.registration_handler + (user_id, token) = yield handler.appservice_register( + user_localpart, as_token + ) + self._remove_session(session) + defer.returnValue({ + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + }) + def _parse_json(request): try: -- cgit 1.4.1 From 3737329d9b40b54bc4205c5f5e7e0946c5e51614 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 10:53:18 +0000 Subject: Handle the fact the list.remove raises if element doesn't exist --- synapse/handlers/federation.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index aba266c2bc..9d0ce9aa50 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -332,8 +332,11 @@ class FederationHandler(BaseHandler): # Try the host we successfully got a response to /make_join/ # request first. - target_hosts.remove(origin) - target_hosts.insert(0, origin) + try: + target_hosts.remove(origin) + target_hosts.insert(0, origin) + except ValueError: + pass ret = yield self.replication_layer.send_join( target_hosts, @@ -521,7 +524,7 @@ class FederationHandler(BaseHandler): "Failed to get destination from event %s", s.event_id ) - destinations.remove(origin) + destinations.discard(origin) logger.debug( "on_send_join_request: Sending event: %s, signatures: %s", @@ -1013,7 +1016,10 @@ class FederationHandler(BaseHandler): for e in missing_remotes: for e_id, _ in e.auth_events: if e_id in missing_remote_ids: - base_remote_rejected.remove(e) + try: + base_remote_rejected.remove(e) + except ValueError: + pass reason_map = {} -- cgit 1.4.1 From 9f2573eea1641806c67d51b98013a17a686c0909 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 10:55:01 +0000 Subject: Return body of response in HttpResponseException --- synapse/http/matrixfederationclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 8559d06b7f..056d446e42 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -169,8 +169,9 @@ class MatrixFederationHttpClient(object): else: # :'( # Update transactions table? + body = yield readBody(response) raise HttpResponseException( - response.code, response.phrase, response + response.code, response.phrase, body ) defer.returnValue(response) -- cgit 1.4.1 From e426df8e106aef0b213928afb6189569474ac5d9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 10:57:14 +0000 Subject: Grant ASes the ability to create alias in their own namespace. Add a new errcode type M_EXCLUSIVE when users try to create aliases inside AS namespaces, and when ASes try to create aliases outside their own namespace. --- synapse/api/auth.py | 12 +++++++++++ synapse/api/errors.py | 3 ++- synapse/handlers/directory.py | 43 ++++++++++++++++++++++++++----------- synapse/rest/client/v1/directory.py | 29 +++++++++++++++---------- 4 files changed, 63 insertions(+), 24 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 4f116184c9..ea8c461729 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -380,6 +380,18 @@ class Auth(object): raise AuthError(403, "Unrecognised access token.", errcode=Codes.UNKNOWN_TOKEN) + @defer.inlineCallbacks + def get_appservice_by_req(self, request): + try: + token = request.args["access_token"][0] + service = yield self.store.get_app_service_by_token(token) + if not service: + raise AuthError(403, "Unrecognised access token.", + errcode=Codes.UNKNOWN_TOKEN) + defer.returnValue(service) + except KeyError: + raise AuthError(403, "Missing access token.") + def is_server_admin(self, user): return self.store.is_server_admin(user) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 5041828f18..eddd889778 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -36,7 +36,8 @@ class Codes(object): CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" CAPTCHA_INVALID = "M_CAPTCHA_INVALID" MISSING_PARAM = "M_MISSING_PARAM", - TOO_LARGE = "M_TOO_LARGE" + TOO_LARGE = "M_TOO_LARGE", + EXCLUSIVE = "M_EXCLUSIVE" class CodeMessageException(RuntimeError): diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 842f075fe0..4c15e57fa6 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -37,24 +37,15 @@ class DirectoryHandler(BaseHandler): ) @defer.inlineCallbacks - def create_association(self, user_id, room_alias, room_id, servers=None): - - # TODO(erikj): Do auth. + def _create_association(self, room_alias, room_id, servers=None): + # general association creation for both human users and app services if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") # TODO(erikj): Change this. # TODO(erikj): Add transactions. - # TODO(erikj): Check if there is a current association. - - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: - raise SynapseError( - 400, "This alias is reserved by an application service." - ) - if not servers: servers = yield self.store.get_joined_hosts_for_room(room_id) @@ -67,6 +58,33 @@ class DirectoryHandler(BaseHandler): servers ) + + @defer.inlineCallbacks + def create_association(self, user_id, room_alias, room_id, servers=None): + # association creation for human users + # TODO(erikj): Do user auth. + + is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) + if is_claimed: + raise SynapseError( + 400, "This alias is reserved by an application service.", + errcode=Codes.EXCLUSIVE + ) + yield self._create_association(room_alias, room_id, servers) + + + @defer.inlineCallbacks + def create_appservice_association(self, service, room_alias, room_id, + servers=None): + if not service.is_interested_in_alias(room_alias.to_string()): + raise SynapseError( + 400, "This application service has not reserved" + " this kind of alias.", errcode=Codes.EXCLUSIVE + ) + + # association creation for app services + yield self._create_association(room_alias, room_id, servers) + @defer.inlineCallbacks def delete_association(self, user_id, room_alias): # TODO Check if server admin @@ -77,7 +95,8 @@ class DirectoryHandler(BaseHandler): is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) if is_claimed: raise SynapseError( - 400, "This alias is reserved by an application service." + 400, "This alias is reserved by an application service.", + errcode=Codes.EXCLUSIVE ) room_id = yield self.store.delete_room_alias(room_alias) diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 8f65efec5f..f7e910bb40 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -45,8 +45,6 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - user, client = yield self.auth.get_user_by_req(request) - content = _parse_json(request) if not "room_id" in content: raise SynapseError(400, "Missing room_id key", @@ -70,16 +68,25 @@ class ClientDirectoryServer(ClientV1RestServlet): dir_handler = self.handlers.directory_handler try: - user_id = user.to_string() - yield dir_handler.create_association( - user_id, room_alias, room_id, servers + # try to auth as a user + user, client = yield self.auth.get_user_by_req(request) + try: + user_id = user.to_string() + yield dir_handler.create_association( + user_id, room_alias, room_id, servers + ) + yield dir_handler.send_room_alias_update_event(user_id, room_id) + except SynapseError as e: + raise e + except: + logger.exception("Failed to create association") + raise + except AuthError: + # try to auth as an application service + service = yield self.auth.get_appservice_by_req(request) + yield dir_handler.create_appservice_association( + service, room_alias, room_id, servers ) - yield dir_handler.send_room_alias_update_event(user_id, room_id) - except SynapseError as e: - raise e - except: - logger.exception("Failed to create association") - raise defer.returnValue((200, {})) -- cgit 1.4.1 From c3ae8def755a043283e945e6653970599d227a43 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 11:32:07 +0000 Subject: Grant ASes the ability to delete aliases in their own namespace. --- synapse/handlers/directory.py | 28 +++++++++++++++++++++++----- synapse/rest/client/v1/directory.py | 31 +++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 4c15e57fa6..c3c95996e7 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -87,10 +87,9 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def delete_association(self, user_id, room_alias): - # TODO Check if server admin + # association deletion for human users - if not self.hs.is_mine(room_alias): - raise SynapseError(400, "Room alias must be local") + # TODO Check if server admin is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) if is_claimed: @@ -99,10 +98,29 @@ class DirectoryHandler(BaseHandler): errcode=Codes.EXCLUSIVE ) + yield self._delete_association(room_alias) + + @defer.inlineCallbacks + def delete_appservice_association(self, service, room_alias): + if not service.is_interested_in_alias(room_alias.to_string()): + raise SynapseError( + 400, + "This application service has not reserved this kind of alias", + errcode=Codes.EXCLUSIVE + ) + yield self._delete_association(room_alias) + + @defer.inlineCallbacks + def _delete_association(self, room_alias): + if not self.hs.is_mine(room_alias): + raise SynapseError(400, "Room alias must be local") + room_id = yield self.store.delete_room_alias(room_alias) - if room_id: - yield self._update_room_alias_events(user_id, room_id) + # TODO - Looks like _update_room_alias_event has never been implemented + # if room_id: + # yield self._update_room_alias_events(user_id, room_id) + @defer.inlineCallbacks def get_association(self, room_alias): diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index f7e910bb40..8e83548bbf 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -87,24 +87,51 @@ class ClientDirectoryServer(ClientV1RestServlet): yield dir_handler.create_appservice_association( service, room_alias, room_id, servers ) + logger.info( + "Application service at %s created alias %s pointing to %s", + service.url, + room_alias.to_string(), + room_id + ) defer.returnValue((200, {})) @defer.inlineCallbacks def on_DELETE(self, request, room_alias): + dir_handler = self.handlers.directory_handler + + try: + service = yield self.auth.get_appservice_by_req(request) + room_alias = RoomAlias.from_string(room_alias) + yield dir_handler.delete_appservice_association( + service, room_alias + ) + logger.info( + "Application service at %s deleted alias %s", + service.url, + room_alias.to_string() + ) + defer.returnValue((200, {})) + except AuthError: + # fallback to default user behaviour if they aren't an AS + pass + user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: raise AuthError(403, "You need to be a server admin") - dir_handler = self.handlers.directory_handler - room_alias = RoomAlias.from_string(room_alias) yield dir_handler.delete_association( user.to_string(), room_alias ) + logger.info( + "User %s deleted alias %s", + user.to_string(), + room_alias.to_string() + ) defer.returnValue((200, {})) -- cgit 1.4.1 From 0995810273ddbc7e5e89b56ad3768fa8f0583bc4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 11:45:19 +0000 Subject: Pyflakes: unused variable. --- synapse/handlers/directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index c3c95996e7..22c6391a15 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -115,7 +115,7 @@ class DirectoryHandler(BaseHandler): if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") - room_id = yield self.store.delete_room_alias(room_alias) + yield self.store.delete_room_alias(room_alias) # TODO - Looks like _update_room_alias_event has never been implemented # if room_id: -- cgit 1.4.1 From c78b5fb1f16957f5398517fa5a99dcb370992b26 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 13:52:16 +0000 Subject: Make seen_ids a set --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9d0ce9aa50..1b709e5e24 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -130,7 +130,9 @@ class FederationHandler(BaseHandler): if auth_chain: event_ids |= {e.event_id for e in auth_chain} - seen_ids = (yield self.store.have_events(event_ids)).keys() + seen_ids = set( + (yield self.store.have_events(event_ids)).keys() + ) if state and auth_chain is not None: # If we have any state or auth_chain given to us by the replication -- cgit 1.4.1 From e890ce223c0c7798efc9961ca1e4c4870cfc7e2f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 14:16:50 +0000 Subject: Don't query auth if the only difference is events that were rejected due to auth. --- synapse/handlers/federation.py | 128 +++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 56 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1b709e5e24..b13b7c7701 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -791,12 +791,12 @@ class FederationHandler(BaseHandler): @log_function def do_auth(self, origin, event, context, auth_events): # Check if we have all the auth events. - res = yield self.store.have_events( + have_events = yield self.store.have_events( [e_id for e_id, _ in event.auth_events] ) event_auth_events = set(e_id for e_id, _ in event.auth_events) - seen_events = set(res.keys()) + seen_events = set(have_events.keys()) missing_auth = event_auth_events - seen_events @@ -839,6 +839,11 @@ class FederationHandler(BaseHandler): auth_events[(e.type, e.state_key)] = e except AuthError: pass + + have_events = yield self.store.have_events( + [e_id for e_id, _ in event.auth_events] + ) + seen_events = set(have_events.keys()) except: # FIXME: logger.exception("Failed to get auth chain") @@ -852,64 +857,75 @@ class FederationHandler(BaseHandler): # Do auth conflict res. logger.debug("Different auth: %s", different_auth) - # 1. Get what we think is the auth chain. - auth_ids = self.auth.compute_auth_events( - event, context.current_state - ) - local_auth_chain = yield self.store.get_auth_chain(auth_ids) - - try: - # 2. Get remote difference. - result = yield self.replication_layer.query_auth( - origin, - event.room_id, - event.event_id, - local_auth_chain, - ) - - seen_remotes = yield self.store.have_events( - [e.event_id for e in result["auth_chain"]] + # Only do auth resolution if we have something new to say. + # We can't rove an auth failure. + do_resolution = False + for e_id in different_auth: + if e_id in have_events: + if have_events[e_id] != RejectedReason.AUTH_ERROR: + do_resolution = True + break + + if do_resolution: + # 1. Get what we think is the auth chain. + auth_ids = self.auth.compute_auth_events( + event, context.current_state ) + local_auth_chain = yield self.store.get_auth_chain(auth_ids) - # 3. Process any remote auth chain events we haven't seen. - for ev in result["auth_chain"]: - if ev.event_id in seen_remotes.keys(): - continue - - if ev.event_id == event.event_id: - continue - - try: - auth_ids = [e_id for e_id, _ in ev.auth_events] - auth = { - (e.type, e.state_key): e for e in result["auth_chain"] - if e.event_id in auth_ids - } - ev.internal_metadata.outlier = True - - logger.debug( - "do_auth %s different_auth: %s", - event.event_id, e.event_id - ) + try: + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) - yield self._handle_new_event( - origin, ev, auth_events=auth - ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) - if ev.event_id in event_auth_events: - auth_events[(ev.type, ev.state_key)] = ev - except AuthError: - pass + # 3. Process any remote auth chain events we haven't seen. + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue + + if ev.event_id == event.event_id: + continue + + try: + auth_ids = [e_id for e_id, _ in ev.auth_events] + auth = { + (e.type, e.state_key): e + for e in result["auth_chain"] + if e.event_id in auth_ids + } + ev.internal_metadata.outlier = True + + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) + + yield self._handle_new_event( + origin, ev, auth_events=auth + ) + + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev + except AuthError: + pass - except: - # FIXME: - logger.exception("Failed to query auth chain") + except: + # FIXME: + logger.exception("Failed to query auth chain") - # 4. Look at rejects and their proofs. - # TODO. + # 4. Look at rejects and their proofs. + # TODO. - context.current_state.update(auth_events) - context.state_group = None + context.current_state.update(auth_events) + context.state_group = None try: self.auth.check(event, auth_events=auth_events) @@ -1028,9 +1044,9 @@ class FederationHandler(BaseHandler): for e in base_remote_rejected: reason = yield self.store.get_rejection_reason(e.event_id) if reason is None: - # FIXME: ERRR?! - logger.warn("Could not find reason for %s", e.event_id) - raise RuntimeError("Could not find reason for %s" % e.event_id) + # TODO: e is not in the current state, so we should + # construct some proof of that. + continue reason_map[e.event_id] = reason -- cgit 1.4.1 From 0cd66885e3ff7828282cc03dd8189763fdb7b927 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 14:38:04 +0000 Subject: Move delta/v13.sql to delta/v12.sql --- synapse/storage/schema/delta/v12.sql | 11 +++++++++++ synapse/storage/schema/delta/v13.sql | 24 ------------------------ 2 files changed, 11 insertions(+), 24 deletions(-) delete mode 100644 synapse/storage/schema/delta/v13.sql (limited to 'synapse') diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql index 16c2258ca4..302d958dbf 100644 --- a/synapse/storage/schema/delta/v12.sql +++ b/synapse/storage/schema/delta/v12.sql @@ -52,3 +52,14 @@ CREATE TABLE IF NOT EXISTS push_rules ( ); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); + +CREATE TABLE IF NOT EXISTS user_filters( + user_id TEXT, + filter_id INTEGER, + filter_json TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); diff --git a/synapse/storage/schema/delta/v13.sql b/synapse/storage/schema/delta/v13.sql deleted file mode 100644 index beb39ca201..0000000000 --- a/synapse/storage/schema/delta/v13.sql +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS user_filters( - user_id TEXT, - filter_id INTEGER, - filter_json TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) -); - -CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( - user_id, filter_id -); -- cgit 1.4.1 From cc0532a4bf328478d508242d830433a33c4f22b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 15:16:26 +0000 Subject: Explicitly list the RejectedReasons that we can prove --- synapse/handlers/federation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b13b7c7701..0f9c82fd06 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -860,9 +860,14 @@ class FederationHandler(BaseHandler): # Only do auth resolution if we have something new to say. # We can't rove an auth failure. do_resolution = False + + provable = [ + RejectedReason.NOT_ANCESTOR, RejectedReason.NOT_ANCESTOR, + ] + for e_id in different_auth: if e_id in have_events: - if have_events[e_id] != RejectedReason.AUTH_ERROR: + if have_events[e_id] in provable: do_resolution = True break -- cgit 1.4.1 From 55a186485c8def40e578201bb33457c9b7df9dde Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 6 Feb 2015 15:58:40 +0000 Subject: SYN-258: get_recent_events_for_room only accepts stream tokens, convert the topological token to a stream token before passing it to get_recent_events_for_room --- synapse/handlers/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 962686f4bb..439164ae39 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -298,15 +298,17 @@ class SyncHandler(BaseHandler): load_limit = max(sync_config.limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key + end_key = room_key while limited and len(recents) < sync_config.limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, - end_token=room_key, + end_token=end_key, ) (room_key, _) = keys + end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents -- cgit 1.4.1 From 20db147ef377cefb2b5cc138c2bd95291e26f3cc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 6 Feb 2015 15:58:40 +0000 Subject: SYN-258: get_recent_events_for_room only accepts stream tokens, convert the topological token to a stream token before passing it to get_recent_events_for_room --- synapse/handlers/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 962686f4bb..439164ae39 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -298,15 +298,17 @@ class SyncHandler(BaseHandler): load_limit = max(sync_config.limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key + end_key = room_key while limited and len(recents) < sync_config.limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, - end_token=room_key, + end_token=end_key, ) (room_key, _) = keys + end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents -- cgit 1.4.1 From c4e3029d55c228375725ccacd326e1ee5cc8dd73 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 16:08:13 +0000 Subject: Add cache layer to state group resolution --- synapse/state.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ synapse/util/__init__.py | 10 +++++++- 2 files changed, 72 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 695a5e7ac4..c45bab5859 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -43,14 +43,30 @@ AuthEventTypes = ( ) +class _StateCacheEntry(object): + def __init__(self, state, state_group, ts): + self.state = state + self.state_group = state_group + self.ts = ts + + class StateHandler(object): """ Responsible for doing state conflict resolution. """ def __init__(self, hs): + self.clock = hs.get_clock() self.store = hs.get_datastore() self.hs = hs + # set of event_ids -> _StateCacheEntry. + self._state_cache = {} + + def f(): + self._prune_cache() + + self.clock.looping_call(f, 10*1000) + @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): """ Returns the current state for the room as a list. This is done by @@ -70,6 +86,11 @@ class StateHandler(object): for e_id, _, _ in events ] + cache = self._state_cache.get(set(event_ids), None) + if cache: + cache.ts = self.clock.time_msec() + defer.returnValue(cache.state_group, cache.state) + res = yield self.resolve_state_groups(event_ids) if event_type: @@ -177,6 +198,11 @@ class StateHandler(object): """ logger.debug("resolve_state_groups event_ids %s", event_ids) + cache = self._state_cache.get(set(event_ids), None) + if cache and cache.state_group: + cache.ts = self.clock.time_msec() + defer.returnValue(cache.state_group, cache.state) + state_groups = yield self.store.get_state_groups( event_ids ) @@ -200,6 +226,14 @@ class StateHandler(object): else: prev_states = [] + cache = _StateCacheEntry( + state=state, + state_group=name, + ts=self.clock.time_msec() + ) + + self._state_cache[set(event_ids)] = cache + defer.returnValue((name, state, prev_states)) state = {} @@ -245,6 +279,14 @@ class StateHandler(object): new_state = unconflicted_state new_state.update(resolved_state) + cache = _StateCacheEntry( + state=new_state, + state_group=None, + ts=self.clock.time_msec() + ) + + self._state_cache[set(event_ids)] = cache + defer.returnValue((None, new_state, prev_states)) @log_function @@ -328,3 +370,24 @@ class StateHandler(object): return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() return sorted(events, key=key_func) + + def _prune_cache(self): + now = self.clock.time_msec() + + if len(self._state_cache) > 100: + sorted_entries = sorted( + self._state_cache.items(), + key=lambda k, v: v.ts, + ) + + for k, _ in sorted_entries[100:]: + self._state_cache.pop(k) + + keys_to_delete = set() + + for key, cache_entry in self._state_cache.items(): + if now - cache_entry.ts > 60*1000: + keys_to_delete.add(key) + + for k in keys_to_delete: + self._state_cache.pop(k) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 4e837a918e..1fd5ba5787 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -15,7 +15,7 @@ from synapse.util.logcontext import LoggingContext -from twisted.internet import reactor +from twisted.internet import reactor, task import time @@ -35,6 +35,14 @@ class Clock(object): """Returns the current system time in miliseconds since epoch.""" return self.time() * 1000 + def looping_call(self, f, msec): + l = task.LoopingCall(f) + l.start(msec/1000.0, now=False) + return l + + def looping_call(self, loop): + loop.stop() + def call_later(self, delay, callback): current_context = LoggingContext.current_context() -- cgit 1.4.1 From b4886264a36ae85bcf922f7d2bc56f0ae59b95e6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 16:17:05 +0000 Subject: Bugfix cache layer --- synapse/state.py | 31 +++++++++++++++++++++---------- synapse/util/__init__.py | 2 +- 2 files changed, 22 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index c45bab5859..7523573f22 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -86,18 +86,19 @@ class StateHandler(object): for e_id, _, _ in events ] - cache = self._state_cache.get(set(event_ids), None) + cache = self._state_cache.get(frozenset(event_ids), None) if cache: cache.ts = self.clock.time_msec() - defer.returnValue(cache.state_group, cache.state) - - res = yield self.resolve_state_groups(event_ids) + state = cache.state + else: + res = yield self.resolve_state_groups(event_ids) + state = res[1] if event_type: - defer.returnValue(res[1].get((event_type, state_key))) + defer.returnValue(state.get((event_type, state_key))) return - defer.returnValue(res[1].values()) + defer.returnValue(state.values()) @defer.inlineCallbacks def compute_event_context(self, event, old_state=None): @@ -198,10 +199,16 @@ class StateHandler(object): """ logger.debug("resolve_state_groups event_ids %s", event_ids) - cache = self._state_cache.get(set(event_ids), None) + cache = self._state_cache.get(frozenset(event_ids), None) if cache and cache.state_group: cache.ts = self.clock.time_msec() - defer.returnValue(cache.state_group, cache.state) + prev_state = cache.state.get((event_type, state_key), None) + if prev_state: + prev_state = prev_state.event_id + prev_states = [prev_state] + else: + prev_states = [] + defer.returnValue((cache.state_group, cache.state, prev_states)) state_groups = yield self.store.get_state_groups( event_ids @@ -232,7 +239,7 @@ class StateHandler(object): ts=self.clock.time_msec() ) - self._state_cache[set(event_ids)] = cache + self._state_cache[frozenset(event_ids)] = cache defer.returnValue((name, state, prev_states)) @@ -285,7 +292,7 @@ class StateHandler(object): ts=self.clock.time_msec() ) - self._state_cache[set(event_ids)] = cache + self._state_cache[frozenset(event_ids)] = cache defer.returnValue((None, new_state, prev_states)) @@ -372,6 +379,8 @@ class StateHandler(object): return sorted(events, key=key_func) def _prune_cache(self): + logger.debug("_prune_cache. before len: ", len(self._state_cache)) + now = self.clock.time_msec() if len(self._state_cache) > 100: @@ -391,3 +400,5 @@ class StateHandler(object): for k in keys_to_delete: self._state_cache.pop(k) + + logger.debug("_prune_cache. after len: ", len(self._state_cache)) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 1fd5ba5787..fee76b0a9b 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -40,7 +40,7 @@ class Clock(object): l.start(msec/1000.0, now=False) return l - def looping_call(self, loop): + def stop_looping_call(self, loop): loop.stop() def call_later(self, delay, callback): -- cgit 1.4.1 From 5bf318e9a60dd05120e9b78b49042337d38ec414 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 16:52:22 +0000 Subject: Bug fixes. --- synapse/app/homeserver.py | 2 ++ synapse/state.py | 82 +++++++++++++++++++++++++++++++---------------- 2 files changed, 57 insertions(+), 27 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f20ccfb5b6..3a128af5f7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -274,6 +274,8 @@ def setup(): hs.get_pusherpool().start() + hs.get_state_handler().start_caching() + if config.daemonize: print config.pid_file daemon = Daemonize( diff --git a/synapse/state.py b/synapse/state.py index 7523573f22..49ac09863e 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -43,6 +43,10 @@ AuthEventTypes = ( ) +SIZE_OF_CACHE = 1000 +EVICTION_TIMEOUT_SECONDS = 20 + + class _StateCacheEntry(object): def __init__(self, state, state_group, ts): self.state = state @@ -59,13 +63,22 @@ class StateHandler(object): self.store = hs.get_datastore() self.hs = hs - # set of event_ids -> _StateCacheEntry. + # dict of set of event_ids -> _StateCacheEntry. + self._state_cache = None + + def start_caching(self): + logger.debug("start_caching") + self._state_cache = {} def f(): - self._prune_cache() + logger.debug("Pruning") + try: + self._prune_cache() + except: + logger.exception("Prune") - self.clock.looping_call(f, 10*1000) + self.clock.looping_call(f, 5*1000) @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): @@ -86,7 +99,10 @@ class StateHandler(object): for e_id, _, _ in events ] - cache = self._state_cache.get(frozenset(event_ids), None) + cache = None + if self._state_cache is not None: + cache = self._state_cache.get(frozenset(event_ids), None) + if cache: cache.ts = self.clock.time_msec() state = cache.state @@ -199,16 +215,19 @@ class StateHandler(object): """ logger.debug("resolve_state_groups event_ids %s", event_ids) - cache = self._state_cache.get(frozenset(event_ids), None) - if cache and cache.state_group: - cache.ts = self.clock.time_msec() - prev_state = cache.state.get((event_type, state_key), None) - if prev_state: - prev_state = prev_state.event_id - prev_states = [prev_state] - else: - prev_states = [] - defer.returnValue((cache.state_group, cache.state, prev_states)) + if self._state_cache is not None: + cache = self._state_cache.get(frozenset(event_ids), None) + if cache and cache.state_group: + cache.ts = self.clock.time_msec() + prev_state = cache.state.get((event_type, state_key), None) + if prev_state: + prev_state = prev_state.event_id + prev_states = [prev_state] + else: + prev_states = [] + defer.returnValue( + (cache.state_group, cache.state, prev_states) + ) state_groups = yield self.store.get_state_groups( event_ids @@ -233,15 +252,16 @@ class StateHandler(object): else: prev_states = [] - cache = _StateCacheEntry( - state=state, - state_group=name, - ts=self.clock.time_msec() - ) + if self._state_cache is not None: + cache = _StateCacheEntry( + state=state, + state_group=name, + ts=self.clock.time_msec() + ) - self._state_cache[frozenset(event_ids)] = cache + self._state_cache[frozenset(event_ids)] = cache - defer.returnValue((name, state, prev_states)) + defer.returnValue((name, state, prev_states)) state = {} for group, g_state in state_groups.items(): @@ -292,7 +312,8 @@ class StateHandler(object): ts=self.clock.time_msec() ) - self._state_cache[frozenset(event_ids)] = cache + if self._state_cache is not None: + self._state_cache[frozenset(event_ids)] = cache defer.returnValue((None, new_state, prev_states)) @@ -379,26 +400,33 @@ class StateHandler(object): return sorted(events, key=key_func) def _prune_cache(self): - logger.debug("_prune_cache. before len: ", len(self._state_cache)) + logger.debug("_prune_cache") + logger.debug( + "_prune_cache. before len: %d", + len(self._state_cache.keys()) + ) now = self.clock.time_msec() - if len(self._state_cache) > 100: + if len(self._state_cache.keys()) > SIZE_OF_CACHE: sorted_entries = sorted( self._state_cache.items(), key=lambda k, v: v.ts, ) - for k, _ in sorted_entries[100:]: + for k, _ in sorted_entries[SIZE_OF_CACHE:]: self._state_cache.pop(k) keys_to_delete = set() for key, cache_entry in self._state_cache.items(): - if now - cache_entry.ts > 60*1000: + if now - cache_entry.ts > EVICTION_TIMEOUT_SECONDS*1000: keys_to_delete.add(key) for k in keys_to_delete: self._state_cache.pop(k) - logger.debug("_prune_cache. after len: ", len(self._state_cache)) + logger.debug( + "_prune_cache. after len: %d", + len(self._state_cache.keys()) + ) -- cgit 1.4.1 From 73a680b2a87b309bf05b4afe2685aaba70201305 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 17:10:04 +0000 Subject: Add errcodes for appservice registrations. --- synapse/handlers/register.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 60821edb05..10ab9bc99e 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -18,7 +18,8 @@ from twisted.internet import defer from synapse.types import UserID from synapse.api.errors import ( - SynapseError, RegistrationError, InvalidCaptchaError, CodeMessageException + AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError, + CodeMessageException ) from ._base import BaseHandler import synapse.util.stringutils as stringutils @@ -111,10 +112,11 @@ class RegistrationHandler(BaseHandler): user_id = user.to_string() service = yield self.store.get_app_service_by_token(as_token) if not service: - raise SynapseError(403, "Invalid application service token.") + raise AuthError(403, "Invalid application service token.") if not service.is_interested_in_user(user_id): raise SynapseError( - 400, "Invalid user localpart for this application service." + 400, "Invalid user localpart for this application service.", + errcode=Codes.EXCLUSIVE ) token = self._generate_token(user_id) yield self.store.register( @@ -181,7 +183,8 @@ class RegistrationHandler(BaseHandler): ] if len(interested_services) > 0: raise SynapseError( - 400, "This user ID is reserved by an application service." + 400, "This user ID is reserved by an application service.", + errcode=Codes.EXCLUSIVE ) def _generate_token(self, user_id): -- cgit 1.4.1 From 34c39398faddae414f50838c744cbc2e4d934a56 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 12:55:13 +0000 Subject: i hate weakly typed languages --- synapse/rest/media/v1/base_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 688e7376ad..d44d5f1298 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -82,7 +82,7 @@ class BaseMediaResource(Resource): raise SynapseError( 404, "Invalid media id token %r" % (request.postpath,), - Codes.UNKKOWN, + Codes.UNKNOWN, ) @staticmethod -- cgit 1.4.1 From e117bc3fc582520e7d7adb86015ad814195cd286 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 12:56:21 +0000 Subject: thou shalt specify a content-length --- synapse/rest/media/v1/media_repository.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 61ed90f39f..9ca4d884dd 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -34,6 +34,7 @@ class MediaRepositoryResource(Resource): => POST /_matrix/media/v1/upload HTTP/1.1 Content-Type: + Content-Length: -- cgit 1.4.1 From 582019f870adbc4a8a8a9ef97b527e0fead77761 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 13:32:14 +0000 Subject: ...and here's the actual impl. git fail. --- synapse/handlers/register.py | 14 ++++++++ synapse/rest/media/v1/upload_resource.py | 57 ++++++++++++++++++-------------- 2 files changed, 46 insertions(+), 25 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 66a89c10b2..2b9d860084 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -99,6 +99,20 @@ class RegistrationHandler(BaseHandler): raise RegistrationError( 500, "Cannot generate user ID.") + # create a default avatar for the user + # XXX: ideally clients would explicitly specify one, but given they don't + # and we want consistent and pretty identicons for random users, we'll + # do it here. + auth_user = UserID.from_string(user_id) + identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) + upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) + content_uri = yield upload_resource.create_content( + "image/png", None, identicon_bytes, len(identicon_bytes), auth_user + ) + profile_handler = self.hs.get_handlers().profile_handler + profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + defer.returnValue((user_id, token)) @defer.inlineCallbacks diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index b939a30e19..5b42782331 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -38,6 +38,35 @@ class UploadResource(BaseMediaResource): def render_OPTIONS(self, request): respond_with_json(request, 200, {}, send_cors=True) return NOT_DONE_YET + + @defer.inlineCallbacks + def create_content(self, media_type, upload_name, content, content_length, auth_user): + media_id = random_string(24) + + fname = self.filepaths.local_media_filepath(media_id) + self._makedirs(fname) + + # This shouldn't block for very long because the content will have + # already been uploaded at this point. + with open(fname, "wb") as f: + f.write(content) + + yield self.store.store_local_media( + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=upload_name, + media_length=content_length, + user_id=auth_user, + ) + media_info = { + "media_type": media_type, + "media_length": content_length, + } + + yield self._generate_local_thumbnails(media_id, media_info) + + defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) @defer.inlineCallbacks def _async_render_POST(self, request): @@ -70,32 +99,10 @@ class UploadResource(BaseMediaResource): # disposition = headers.getRawHeaders("Content-Disposition")[0] # TODO(markjh): parse content-dispostion - media_id = random_string(24) - - fname = self.filepaths.local_media_filepath(media_id) - self._makedirs(fname) - - # This shouldn't block for very long because the content will have - # already been uploaded at this point. - with open(fname, "wb") as f: - f.write(request.content.read()) - - yield self.store.store_local_media( - media_id=media_id, - media_type=media_type, - time_now_ms=self.clock.time_msec(), - upload_name=None, - media_length=content_length, - user_id=auth_user, + content_uri = yield self.create_content( + media_type, None, request.content.read(), + content_length, auth_user ) - media_info = { - "media_type": media_type, - "media_length": content_length, - } - - yield self._generate_local_thumbnails(media_id, media_info) - - content_uri = "mxc://%s/%s" % (self.server_name, media_id) respond_with_json( request, 200, {"content_uri": content_uri}, send_cors=True -- cgit 1.4.1 From 37b6b880ef084f2cdd3cce1a275aa46764131c91 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 21:24:08 +0000 Subject: don't give up if we can't create default avatars during tests --- synapse/handlers/register.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 2b9d860084..4f06c487b1 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -103,15 +103,18 @@ class RegistrationHandler(BaseHandler): # XXX: ideally clients would explicitly specify one, but given they don't # and we want consistent and pretty identicons for random users, we'll # do it here. - auth_user = UserID.from_string(user_id) - identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) - upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) - identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) - content_uri = yield upload_resource.create_content( - "image/png", None, identicon_bytes, len(identicon_bytes), auth_user - ) - profile_handler = self.hs.get_handlers().profile_handler - profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + try: + auth_user = UserID.from_string(user_id) + identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) + upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) + content_uri = yield upload_resource.create_content( + "image/png", None, identicon_bytes, len(identicon_bytes), auth_user + ) + profile_handler = self.hs.get_handlers().profile_handler + profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + except NotImplementedError: + pass # make tests pass without messing around creating default avatars defer.returnValue((user_id, token)) -- cgit 1.4.1 From 8be07e0db456d9c79d258c7a51482442445fad00 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 8 Feb 2015 00:34:11 +0000 Subject: kill off fnmatch in favour of word-boundary based push alerts (untested) --- synapse/push/__init__.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 7293715293..8e11abfa5c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -22,7 +22,6 @@ import synapse.util.async import baserules import logging -import fnmatch import json import re @@ -130,26 +129,35 @@ class Pusher(object): defer.returnValue(Pusher.DEFAULT_ACTIONS) + @staticmethod + def _glob_to_regexp(glob): + r = re.escape(glob) + r = re.sub(r'\\\*', r'.*', r) + r = re.sub(r'\\\?', r'.', r) + + # handle [abc], [a-z] and [!a-z] style ranges. + r = re.sub(r'\\\[(\\\!|)(.*)\\\]', + lambda x: ('[%s%s]' % (x.group(1) and '^' or '', + re.sub(r'\\\-', '-', x.group(2)))), r) + return r + def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): if condition['kind'] == 'event_match': if 'pattern' not in condition: logger.warn("event_match condition with no pattern") return False - pat = condition['pattern'] - - if pat.strip("*?[]") == pat: - # no special glob characters so we assume the user means - # 'contains this string' rather than 'is this string' - pat = "*%s*" % (pat,) - + # XXX: optimisation: cache our pattern regexps + r = r'\b%s\b' % _glob_to_regexp(condition['pattern']) val = _value_for_dotted_key(condition['key'], ev) if val is None: return False - return fnmatch.fnmatch(val.upper(), pat.upper()) + return re.match(r, val, flags=re.IGNORECASE) != None + elif condition['kind'] == 'device': if 'profile_tag' not in condition: return True return condition['profile_tag'] == self.profile_tag + elif condition['kind'] == 'contains_display_name': # This is special because display names can be different # between rooms and so you can't really hard code it in a rule. @@ -159,9 +167,9 @@ class Pusher(object): return False if not display_name: return False - return fnmatch.fnmatch( - ev['content']['body'].upper(), "*%s*" % (display_name.upper(),) - ) + return re.match("\b%s\b" % re.escape(display_name), + ev['content']['body'], flags=re.IGNORECASE) != None + elif condition['kind'] == 'room_member_count': if 'is' not in condition: return False -- cgit 1.4.1 From c2afc2ad90b1121a072765237f5c4dc8d765ddf7 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 8 Feb 2015 00:37:03 +0000 Subject: oops --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8e11abfa5c..7a41c5ece3 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -147,7 +147,7 @@ class Pusher(object): logger.warn("event_match condition with no pattern") return False # XXX: optimisation: cache our pattern regexps - r = r'\b%s\b' % _glob_to_regexp(condition['pattern']) + r = r'\b%s\b' % self._glob_to_regexp(condition['pattern']) val = _value_for_dotted_key(condition['key'], ev) if val is None: return False -- cgit 1.4.1 From ecb0f7806305b3433d8c2a5a5bb413226f2e90f8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 8 Feb 2015 02:37:35 +0000 Subject: glob *s should probably be non-greedy --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 7a41c5ece3..07b5f0187c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -132,7 +132,7 @@ class Pusher(object): @staticmethod def _glob_to_regexp(glob): r = re.escape(glob) - r = re.sub(r'\\\*', r'.*', r) + r = re.sub(r'\\\*', r'.*?', r) r = re.sub(r'\\\?', r'.', r) # handle [abc], [a-z] and [!a-z] style ranges. -- cgit 1.4.1 From ac3183caaa66b750996d90c0ac9ed430f623909c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 12:03:37 +0000 Subject: Register a user account for the AS when the AS registers. Add 'sender' column to AS table. --- synapse/appservice/__init__.py | 3 ++- synapse/handlers/appservice.py | 8 +++++++- synapse/storage/appservice.py | 5 +++-- synapse/storage/schema/application_services.sql | 1 + synapse/storage/schema/delta/v14.sql | 1 + 5 files changed, 14 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 46d46a5a48..fb9bfffe5d 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -35,10 +35,11 @@ class ApplicationService(object): NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None, hs_token=None, - txn_id=None): + sender=None, txn_id=None): self.token = token self.url = url self.hs_token = hs_token + self.sender = sender self.namespaces = self._check_namespaces(namespaces) self.txn_id = txn_id diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index fa810b9a98..5071a12eb1 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -52,8 +52,14 @@ class ApplicationServicesHandler(object): "Consult the home server admin.", errcode=Codes.FORBIDDEN ) - logger.info("Updating application service info...") + app_service.hs_token = self._generate_hs_token() + + # create a sender for this application service which is used when + # creating rooms, etc.. + account = yield self.hs.get_handlers().registration_handler.register() + app_service.sender = account[0] + yield self.store.update_app_service(app_service) defer.returnValue(app_service) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 3c8bf9ad0d..eef77e737e 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -130,8 +130,9 @@ class ApplicationServiceStore(SQLBaseStore): return False txn.execute( - "UPDATE application_services SET url=?, hs_token=? WHERE id=?", - (service.url, service.hs_token, as_id,) + "UPDATE application_services SET url=?, hs_token=?, sender=? " + "WHERE id=?", + (service.url, service.hs_token, service.sender, as_id,) ) # cleanup regex txn.execute( diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql index 03b5a10c8a..e491ad5aec 100644 --- a/synapse/storage/schema/application_services.sql +++ b/synapse/storage/schema/application_services.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS application_services( url TEXT, token TEXT, hs_token TEXT, + sender TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); diff --git a/synapse/storage/schema/delta/v14.sql b/synapse/storage/schema/delta/v14.sql index 03b5a10c8a..e491ad5aec 100644 --- a/synapse/storage/schema/delta/v14.sql +++ b/synapse/storage/schema/delta/v14.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS application_services( url TEXT, token TEXT, hs_token TEXT, + sender TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); -- cgit 1.4.1 From 24cc6979fb384ef383309b27d06985ba3a845b2b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 13:46:22 +0000 Subject: Log when we receive a request, when we send a response and how long it took to process it. --- synapse/app/homeserver.py | 2 +- synapse/http/server.py | 23 +++++++++++++++++++++-- synapse/rest/client/v1/__init__.py | 2 +- synapse/rest/client/v2_alpha/__init__.py | 2 +- 4 files changed, 24 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f20ccfb5b6..0f175ec3f4 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -67,7 +67,7 @@ class SynapseHomeServer(HomeServer): return ClientV2AlphaRestResource(self) def build_resource_for_federation(self): - return JsonResource() + return JsonResource(self) def build_resource_for_web_client(self): syweb_path = os.path.dirname(syweb.__file__) diff --git a/synapse/http/server.py b/synapse/http/server.py index 0f6539e1be..6d084fa33c 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -69,9 +69,10 @@ class JsonResource(HttpServer, resource.Resource): _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"]) - def __init__(self): + def __init__(self, hs): resource.Resource.__init__(self) + self.clock = hs.get_clock() self.path_regexs = {} def register_path(self, method, path_pattern, callback): @@ -111,6 +112,7 @@ class JsonResource(HttpServer, resource.Resource): This checks if anyone has registered a callback for that method and path. """ + code = None try: # Just say yes to OPTIONS. if request.method == "OPTIONS": @@ -130,6 +132,13 @@ class JsonResource(HttpServer, resource.Resource): urllib.unquote(u).decode("UTF-8") for u in m.groups() ] + logger.info( + "Received request: %s %s", + request.method, request.path + ) + + start = self.clock.time_msec() + code, response = yield path_entry.callback( request, *args @@ -145,9 +154,11 @@ class JsonResource(HttpServer, resource.Resource): logger.info("%s SynapseError: %s - %s", request, e.code, e.msg) else: logger.exception(e) + + code = e.code self._send_response( request, - e.code, + code, cs_exception(e), response_code_message=e.response_code_message ) @@ -158,6 +169,14 @@ class JsonResource(HttpServer, resource.Resource): 500, {"error": "Internal server error"} ) + finally: + code = str(code) if code else "-" + + end = self.clock.time_msec() + logger.info( + "Processed request: %dms %s %s %s", + end-start, code, request.method, request.path + ) def _send_response(self, request, code, response_json_object, response_code_message=None): diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index d8d01cdd16..21876b3487 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -25,7 +25,7 @@ class ClientV1RestResource(JsonResource): """A resource for version 1 of the matrix client API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 8f611de3a8..bca65f2a6a 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -25,7 +25,7 @@ class ClientV2AlphaRestResource(JsonResource): """A resource for version 2 alpha of the matrix client API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod -- cgit 1.4.1 From 5a7dd058184613c70041a61fdbc2ccce104bb500 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 14:14:15 +0000 Subject: Modify auth.get_user_by_req for authing appservices directly. Add logic to map the appservice token to the autogenned appservice user ID. Add unit tests for all forms of get_user_by_req (user/appservice, valid/bad/missing tokens) --- synapse/api/auth.py | 34 ++++---- synapse/storage/appservice.py | 4 +- tests/api/test_auth.py | 139 +++++++++++++++++++++++++++++++++ tests/rest/client/v1/test_presence.py | 3 + tests/rest/client/v2_alpha/__init__.py | 4 +- 5 files changed, 164 insertions(+), 20 deletions(-) create mode 100644 tests/api/test_auth.py (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index ea8c461729..310a428066 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -302,27 +302,26 @@ class Auth(object): # Check for application service tokens with a user_id override try: - if "user_id" not in request.args: - # This has to be done like this rather than relying on it - # natively throwing because tests use a Mock for the request - # object which doesn't throw :/ - raise KeyError - - masquerade_user_id = request.args["user_id"][0] app_service = yield self.store.get_app_service_by_token( access_token ) if not app_service: - raise AuthError( - 403, "Invalid application service access token" - ) - if not app_service.is_interested_in_user(masquerade_user_id): - raise AuthError( - 403, - "Application service cannot masquerade as this user." - ) + raise KeyError + + user_id = app_service.sender + if "user_id" in request.args: + user_id = request.args["user_id"][0] + if not app_service.is_interested_in_user(user_id): + raise AuthError( + 403, + "Application service cannot masquerade as this user." + ) + + if not user_id: + raise KeyError + defer.returnValue( - (UserID.from_string(masquerade_user_id), ClientInfo("", "")) + (UserID.from_string(user_id), ClientInfo("", "")) ) return except KeyError: @@ -366,8 +365,7 @@ class Auth(object): try: ret = yield self.store.get_user_by_token(token=token) if not ret: - raise StoreError() - + raise StoreError(400, "Unknown token") user_info = { "admin": bool(ret.get("admin", False)), "device_id": ret.get("device_id"), diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index eef77e737e..ba31c68595 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -215,6 +215,7 @@ class ApplicationServiceStore(SQLBaseStore): "url": res["url"], "token": as_token, "hs_token": res["hs_token"], + "sender": res["sender"], "namespaces": { ApplicationService.NS_USERS: [], ApplicationService.NS_ALIASES: [], @@ -240,6 +241,7 @@ class ApplicationServiceStore(SQLBaseStore): token=service["token"], url=service["url"], namespaces=service["namespaces"], - hs_token=service["hs_token"] + hs_token=service["hs_token"], + sender=service["sender"] )) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py new file mode 100644 index 0000000000..1d8367ce42 --- /dev/null +++ b/tests/api/test_auth.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from tests import unittest +from twisted.internet import defer + +from mock import Mock, NonCallableMock + +from synapse.api.auth import Auth +from synapse.api.errors import AuthError +from synapse.types import UserID + +class AuthTestCase(unittest.TestCase): + + def setUp(self): + self.state_handler = Mock() + self.store = Mock() + + self.hs = Mock() + self.hs.get_datastore = Mock(return_value=self.store) + self.hs.get_state_handler = Mock(return_value=self.state_handler) + self.auth = Auth(self.hs) + + self.test_user = "@foo:bar" + self.test_token = "_test_token_" + + @defer.inlineCallbacks + def test_get_user_by_req_user_valid_token(self): + self.store.get_app_service_by_token = Mock(return_value=None) + user_info = { + "name": self.test_user, + "device_id": "nothing", + "token_id": "ditto", + "admin": False + } + self.store.get_user_by_token = Mock(return_value=user_info) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + (user, info) = yield self.auth.get_user_by_req(request) + self.assertEquals(user.to_string(), self.test_user) + + def test_get_user_by_req_user_bad_token(self): + self.store.get_app_service_by_token = Mock(return_value=None) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + d = self.auth.get_user_by_req(request) + self.failureResultOf(d, AuthError) + + def test_get_user_by_req_user_missing_token(self): + self.store.get_app_service_by_token = Mock(return_value=None) + user_info = { + "name": self.test_user, + "device_id": "nothing", + "token_id": "ditto", + "admin": False + } + self.store.get_user_by_token = Mock(return_value=user_info) + + request = Mock(args={}) + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + d = self.auth.get_user_by_req(request) + self.failureResultOf(d, AuthError) + + @defer.inlineCallbacks + def test_get_user_by_req_appservice_valid_token(self): + app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + self.store.get_app_service_by_token = Mock(return_value=app_service) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + (user, info) = yield self.auth.get_user_by_req(request) + self.assertEquals(user.to_string(), self.test_user) + + def test_get_user_by_req_appservice_bad_token(self): + self.store.get_app_service_by_token = Mock(return_value=None) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + d = self.auth.get_user_by_req(request) + self.failureResultOf(d, AuthError) + + def test_get_user_by_req_appservice_missing_token(self): + app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + self.store.get_app_service_by_token = Mock(return_value=app_service) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + d = self.auth.get_user_by_req(request) + self.failureResultOf(d, AuthError) + + @defer.inlineCallbacks + def test_get_user_by_req_appservice_valid_token_valid_user_id(self): + masquerading_user_id = "@doppelganger:matrix.org" + app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + app_service.is_interested_in_user = Mock(return_value=True) + self.store.get_app_service_by_token = Mock(return_value=app_service) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.args["user_id"] = [masquerading_user_id] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + (user, info) = yield self.auth.get_user_by_req(request) + self.assertEquals(user.to_string(), masquerading_user_id) + + def test_get_user_by_req_appservice_valid_token_bad_user_id(self): + masquerading_user_id = "@doppelganger:matrix.org" + app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + app_service.is_interested_in_user = Mock(return_value=False) + self.store.get_app_service_by_token = Mock(return_value=app_service) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.args["user_id"] = [masquerading_user_id] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + d = self.auth.get_user_by_req(request) + self.failureResultOf(d, AuthError) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index f849120a3e..e5d876d89a 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -65,6 +65,7 @@ class PresenceStateTestCase(unittest.TestCase): hs.handlers = JustPresenceHandlers(hs) self.datastore = hs.get_datastore() + self.datastore.get_app_service_by_token = Mock(return_value=None) def get_presence_list(*a, **kw): return defer.succeed([]) @@ -154,6 +155,7 @@ class PresenceListTestCase(unittest.TestCase): hs.handlers = JustPresenceHandlers(hs) self.datastore = hs.get_datastore() + self.datastore.get_app_service_by_token = Mock(return_value=None) def has_presence_state(user_localpart): return defer.succeed( @@ -303,6 +305,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.handlers.room_member_handler.get_rooms_for_user = get_rooms_for_user self.mock_datastore = hs.get_datastore() + self.mock_datastore.get_app_service_by_token = Mock(return_value=None) def get_profile_displayname(user_id): return defer.succeed("Frank") diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index fa70575c57..7c2b0dfa0e 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -59,6 +59,8 @@ class V2AlphaRestTestCase(unittest.TestCase): r.register_servlets(hs, self.mock_resource) def make_datastore_mock(self): - return Mock(spec=[ + store = Mock(spec=[ "insert_client_ip", ]) + store.get_app_service_by_token = Mock(return_value=None) + return store -- cgit 1.4.1 From 784d714a3f2be2cf15151eee8723377a0e3eea11 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 9 Feb 2015 14:17:52 +0000 Subject: Fix server default rule injection (downwards, not upwards!) --- synapse/push/baserules.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 8d4b806da6..37878f1e0b 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -4,24 +4,24 @@ def list_with_base_rules(rawrules, user_name): ruleslist = [] # shove the server default rules for each kind onto the end of each - current_prio_class = 1 + current_prio_class = PRIORITY_CLASS_INVERSE_MAP.keys()[-1] for r in rawrules: - if r['priority_class'] > current_prio_class: - while current_prio_class < r['priority_class']: + if r['priority_class'] < current_prio_class: + while r['priority_class'] < current_prio_class: ruleslist.extend(make_base_rules( user_name, PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) ) - current_prio_class += 1 + current_prio_class -= 1 ruleslist.append(r) - while current_prio_class <= PRIORITY_CLASS_INVERSE_MAP.keys()[-1]: + while current_prio_class > 0: ruleslist.extend(make_base_rules( user_name, PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) ) - current_prio_class += 1 + current_prio_class -= 1 return ruleslist -- cgit 1.4.1 From 75656712e34694460ce7b12fc5a467667e04ea21 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 14:22:52 +0000 Subject: Time how long we're spending on the database thread --- synapse/app/homeserver.py | 2 ++ synapse/storage/_base.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0f175ec3f4..8976ff2e82 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -274,6 +274,8 @@ def setup(): hs.get_pusherpool().start() + hs.get_datastore().start_profiling() + if config.daemonize: print config.pid_file daemon = Daemonize( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index b350fd61f1..0849c5f1b4 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -85,6 +85,28 @@ class SQLBaseStore(object): self._db_pool = hs.get_db_pool() self._clock = hs.get_clock() + self._previous_txn_total_time = 0 + self._current_txn_total_time = 0 + self._previous_loop_ts = 0 + + def start_profiling(self): + self._previous_loop_ts = self._clock.time_msec() + + def loop(): + curr = self._current_txn_total_time + prev = self._previous_txn_total_time + self._previous_txn_total_time = curr + + time_now = self._clock.time_msec() + time_then = self._previous_loop_ts + self._previous_loop_ts = time_now + + ratio = (curr - prev)/(time_now - time_then) + + logger.info("Total database time: %.3f", ratio) + + self._clock.looping_call(loop, 1000) + @defer.inlineCallbacks def runInteraction(self, desc, func, *args, **kwargs): """Wraps the .runInteraction() method on the underlying db_pool.""" @@ -114,6 +136,9 @@ class SQLBaseStore(object): "[TXN END] {%s} %f", name, end - start ) + + self._current_txn_total_time += end - start + with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( inner_func, *args, **kwargs -- cgit 1.4.1 From 66fde49f071d75ea8bfdfac02fd4fa6fab5a9bf4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 14:45:15 +0000 Subject: Log database time every 10s and log as percentage --- synapse/storage/_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 0849c5f1b4..f1df5d39fd 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -103,9 +103,9 @@ class SQLBaseStore(object): ratio = (curr - prev)/(time_now - time_then) - logger.info("Total database time: %.3f", ratio) + logger.info("Total database time: %.3f%", ratio * 100) - self._clock.looping_call(loop, 1000) + self._clock.looping_call(loop, 10000) @defer.inlineCallbacks def runInteraction(self, desc, func, *args, **kwargs): -- cgit 1.4.1 From ef995e69460a117e78a72bcef285f9a0c7438487 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 14:47:59 +0000 Subject: Add looping_call to Clock --- synapse/util/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 4e837a918e..fee76b0a9b 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -15,7 +15,7 @@ from synapse.util.logcontext import LoggingContext -from twisted.internet import reactor +from twisted.internet import reactor, task import time @@ -35,6 +35,14 @@ class Clock(object): """Returns the current system time in miliseconds since epoch.""" return self.time() * 1000 + def looping_call(self, f, msec): + l = task.LoopingCall(f) + l.start(msec/1000.0, now=False) + return l + + def stop_looping_call(self, loop): + loop.stop() + def call_later(self, delay, callback): current_context = LoggingContext.current_context() -- cgit 1.4.1 From c4ee4ce93ec6075bc076b12520fd72769079f37c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 15:00:37 +0000 Subject: Fix typo --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f1df5d39fd..310ee0104c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -103,7 +103,7 @@ class SQLBaseStore(object): ratio = (curr - prev)/(time_now - time_then) - logger.info("Total database time: %.3f%", ratio * 100) + logger.info("Total database time: %.3f%%", ratio * 100) self._clock.looping_call(loop, 10000) -- cgit 1.4.1 From f7cac2f7b699fb3bb58c02fa1828aeec6c319f01 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 15:01:28 +0000 Subject: Fix bugs so lazy room joining works as intended. --- synapse/appservice/api.py | 6 +++--- synapse/handlers/appservice.py | 6 +++--- synapse/handlers/directory.py | 23 +++++++++++++++++------ 3 files changed, 23 insertions(+), 12 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 9cce4e0973..15ac1e27fc 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -41,7 +41,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": service.hs_token }) - if response: # just an empty json object + if response is not None: # just an empty json object defer.returnValue(True) except CodeMessageException as e: if e.code == 404: @@ -60,13 +60,13 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": service.hs_token }) - if response: # just an empty json object + if response is not None: # just an empty json object defer.returnValue(True) except CodeMessageException as e: + logger.warning("query_alias to %s received %s", uri, e.code) if e.code == 404: defer.returnValue(False) return - logger.warning("query_alias to %s received %s", uri, e.code) except Exception as ex: logger.warning("query_alias to %s threw exception %s", uri, ex) defer.returnValue(False) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 5071a12eb1..8591a77bf3 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -124,15 +124,15 @@ class ApplicationServicesHandler(object): namedtuple: with keys "room_id" and "servers" or None if no association can be found. """ - room_alias = room_alias.to_string() + room_alias_str = room_alias.to_string() alias_query_services = yield self._get_services_for_event( event=None, restrict_to=ApplicationService.NS_ALIASES, - alias_list=[room_alias] + alias_list=[room_alias_str] ) for alias_service in alias_query_services: is_known_alias = yield self.appservice_api.query_alias( - alias_service, room_alias + alias_service, room_alias_str ) if is_known_alias: # the alias exists now so don't query more ASes. diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 22c6391a15..87bc12c983 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -64,8 +64,11 @@ class DirectoryHandler(BaseHandler): # association creation for human users # TODO(erikj): Do user auth. - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: + can_create = yield self.can_modify_alias( + room_alias, + user_id=user_id + ) + if not can_create: raise SynapseError( 400, "This alias is reserved by an application service.", errcode=Codes.EXCLUSIVE @@ -91,8 +94,11 @@ class DirectoryHandler(BaseHandler): # TODO Check if server admin - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: + can_delete = yield self.can_modify_alias( + room_alias, + user_id=user_id + ) + if not can_delete: raise SynapseError( 400, "This alias is reserved by an application service.", errcode=Codes.EXCLUSIVE @@ -228,9 +234,14 @@ class DirectoryHandler(BaseHandler): defer.returnValue(result) @defer.inlineCallbacks - def is_alias_exclusive_to_appservices(self, alias): + def can_modify_alias(self, alias, user_id=None): services = yield self.store.get_app_services() interested_services = [ s for s in services if s.is_interested_in_alias(alias.to_string()) ] - defer.returnValue(len(interested_services) > 0) + for service in interested_services: + if user_id == service.sender: + # this user IS the app service + defer.returnValue(True) + return + defer.returnValue(len(interested_services) == 0) -- cgit 1.4.1 From a578251b4800f20424e8b294a42cc6c65ef568a2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 9 Feb 2015 16:44:47 +0000 Subject: only do word-boundary patches on bodies for now --- synapse/push/__init__.py | 5 ++++- synapse/python_dependencies.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 07b5f0187c..58c8cf700b 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -147,7 +147,10 @@ class Pusher(object): logger.warn("event_match condition with no pattern") return False # XXX: optimisation: cache our pattern regexps - r = r'\b%s\b' % self._glob_to_regexp(condition['pattern']) + if condition['key'] == 'content.body': + r = r'\b%s\b' % self._glob_to_regexp(condition['pattern']) + else: + r = r'^%s$' % self._glob_to_regexp(condition['pattern']) val = _value_for_dotted_key(condition['key'], ev) if val is None: return False diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a89d618606..f429d1beda 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted==14.0.2": ["twisted==14.0.2"], + "Twisted==14.0.0": ["twisted==14.0.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], -- cgit 1.4.1 From bd2373277de8fd571d8eb0af38b107065e567a02 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 9 Feb 2015 16:48:09 +0000 Subject: oops --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index f429d1beda..44d1a0404d 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted==14.0.0": ["twisted==14.0.0"], + "Twisted==14.0.0": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], -- cgit 1.4.1 From 0b725f5c4f8f0550205c09bbdc775f6ad613a1fd Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 9 Feb 2015 16:48:31 +0000 Subject: oops --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 44d1a0404d..a89d618606 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted==14.0.0": ["twisted==14.0.2"], + "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], -- cgit 1.4.1 From 8f616684a3a712ccf06349c67bb64779f06d8da1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 9 Feb 2015 17:01:40 +0000 Subject: Need to use re.search if looking for matches not at the start of the string. Also comparisons with None should be 'is'. --- synapse/push/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 58c8cf700b..6f143a5df9 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -154,7 +154,7 @@ class Pusher(object): val = _value_for_dotted_key(condition['key'], ev) if val is None: return False - return re.match(r, val, flags=re.IGNORECASE) != None + return re.search(r, val, flags=re.IGNORECASE) is not None elif condition['kind'] == 'device': if 'profile_tag' not in condition: @@ -170,8 +170,8 @@ class Pusher(object): return False if not display_name: return False - return re.match("\b%s\b" % re.escape(display_name), - ev['content']['body'], flags=re.IGNORECASE) != None + return re.search("\b%s\b" % re.escape(display_name), + ev['content']['body'], flags=re.IGNORECASE) is not None elif condition['kind'] == 'room_member_count': if 'is' not in condition: -- cgit 1.4.1 From d94f682a4c3e7bab5079d516582f0ee44a3d3f06 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 17:41:29 +0000 Subject: During room intial sync, only calculate current state once. --- synapse/api/auth.py | 21 ++++++++++++++------- synapse/handlers/message.py | 32 ++++++++++++++++++++++---------- synapse/handlers/sync.py | 9 ++++++--- synapse/state.py | 2 +- 4 files changed, 43 insertions(+), 21 deletions(-) (limited to 'synapse') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 7105ee21dc..0054745363 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -89,12 +89,19 @@ class Auth(object): raise @defer.inlineCallbacks - def check_joined_room(self, room_id, user_id): - member = yield self.state.get_current_state( - room_id=room_id, - event_type=EventTypes.Member, - state_key=user_id - ) + def check_joined_room(self, room_id, user_id, current_state=None): + if current_state: + member = current_state.get( + (EventTypes.Member, user_id), + None + ) + else: + member = yield self.state.get_current_state( + room_id=room_id, + event_type=EventTypes.Member, + state_key=user_id + ) + self._check_joined_room(member, user_id, room_id) defer.returnValue(member) @@ -102,7 +109,7 @@ class Auth(object): def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) - for event in curr_state: + for event in curr_state.values(): if event.type == EventTypes.Member: try: if UserID.from_string(event.state_key).domain != host: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3f51f38f18..3355adefcf 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -35,6 +35,7 @@ class MessageHandler(BaseHandler): def __init__(self, hs): super(MessageHandler, self).__init__(hs) self.hs = hs + self.state = hs.get_state_handler() self.clock = hs.get_clock() self.validator = EventValidator() @@ -225,7 +226,9 @@ class MessageHandler(BaseHandler): # TODO: This is duplicating logic from snapshot_all_rooms current_state = yield self.state_handler.get_current_state(room_id) now = self.clock.time_msec() - defer.returnValue([serialize_event(c, now) for c in current_state]) + defer.returnValue( + [serialize_event(c, now) for c in current_state.values()] + ) @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, @@ -313,7 +316,7 @@ class MessageHandler(BaseHandler): ) d["state"] = [ serialize_event(c, time_now, as_client_event) - for c in current_state + for c in current_state.values() ] except: logger.exception("Failed to get snapshot") @@ -329,7 +332,14 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def room_initial_sync(self, user_id, room_id, pagin_config=None, feedback=False): - yield self.auth.check_joined_room(room_id, user_id) + current_state = yield self.state.get_current_state( + room_id=room_id, + ) + + yield self.auth.check_joined_room( + room_id, user_id, + current_state=current_state + ) # TODO(paul): I wish I was called with user objects not user_id # strings... @@ -337,13 +347,12 @@ class MessageHandler(BaseHandler): # TODO: These concurrently time_now = self.clock.time_msec() - state_tuples = yield self.state_handler.get_current_state(room_id) - state = [serialize_event(x, time_now) for x in state_tuples] + state = [ + serialize_event(x, time_now) + for x in current_state.values() + ] - member_event = (yield self.store.get_room_member( - user_id=user_id, - room_id=room_id - )) + member_event = current_state.get((EventTypes.Member, user_id,)) now_token = yield self.hs.get_event_sources().get_current_token() @@ -360,7 +369,10 @@ class MessageHandler(BaseHandler): start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) - room_members = yield self.store.get_room_members(room_id) + room_members = [ + m for m in current_state.values() + if m.type == EventTypes.Member + ] presence_handler = self.hs.get_handlers().presence_handler presence = [] diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 439164ae39..5af90cc5d1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -175,9 +175,10 @@ class SyncHandler(BaseHandler): room_id, sync_config, now_token, ) - current_state_events = yield self.state_handler.get_current_state( + current_state = yield self.state_handler.get_current_state( room_id ) + current_state_events = current_state.values() defer.returnValue(RoomSyncResult( room_id=room_id, @@ -347,9 +348,10 @@ class SyncHandler(BaseHandler): # TODO(mjark): This seems racy since this isn't being passed a # token to indicate what point in the stream this is - current_state_events = yield self.state_handler.get_current_state( + current_state = yield self.state_handler.get_current_state( room_id ) + current_state_events = current_state.values() state_at_previous_sync = yield self.get_state_at_previous_sync( room_id, since_token=since_token @@ -431,6 +433,7 @@ class SyncHandler(BaseHandler): joined = True if joined: - state_delta = yield self.state_handler.get_current_state(room_id) + res = yield self.state_handler.get_current_state(room_id) + state_delta = res.values() defer.returnValue(state_delta) diff --git a/synapse/state.py b/synapse/state.py index 695a5e7ac4..54380b9e5c 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -76,7 +76,7 @@ class StateHandler(object): defer.returnValue(res[1].get((event_type, state_key))) return - defer.returnValue(res[1].values()) + defer.returnValue(res[1]) @defer.inlineCallbacks def compute_event_context(self, event, old_state=None): -- cgit 1.4.1 From 3a5ad7dbd5a375023c96ee65c901f8be5ab02341 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 17:55:56 +0000 Subject: Performance counters for database transaction names --- synapse/storage/_base.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 310ee0104c..bcb03cbdcb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -39,6 +39,7 @@ class LoggingTransaction(object): passed to the constructor. Adds logging to the .execute() method.""" __slots__ = ["txn", "name"] + def __init__(self, txn, name): object.__setattr__(self, "txn", txn) object.__setattr__(self, "name", name) @@ -88,6 +89,8 @@ class SQLBaseStore(object): self._previous_txn_total_time = 0 self._current_txn_total_time = 0 self._previous_loop_ts = 0 + self._txn_perf_counters = {} + self._previous_txn_perf_counters = {} def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -103,7 +106,29 @@ class SQLBaseStore(object): ratio = (curr - prev)/(time_now - time_then) - logger.info("Total database time: %.3f%%", ratio * 100) + txn_counters = [] + for name, (count, cum_time) in self._txn_perf_counters.items(): + prev_count, prev_time = self._previous_txn_perf_counters.get( + name, (0,0) + ) + txn_counters.append(( + (cum_time - prev_time) / (time_now - time_then), + count - prev_count, + name + )) + + self._previous_txn_perf_counters = dict(self._txn_perf_counters) + + txn_counters.sort(reverse=True) + top_three_counters = ", ".join( + "%s(%d): %.3f%%" % (name, count, 100 * ratio) + for ratio, count, name in txn_counters[:3] + ) + + logger.info( + "Total database time: %.3f%% {%s}", + ratio * 100, top_three_counters + ) self._clock.looping_call(loop, 10000) @@ -139,6 +164,11 @@ class SQLBaseStore(object): self._current_txn_total_time += end - start + count, cum_time = self._txn_perf_counters.get(name, (0,0)) + count += 1 + cum_time += end - start + self._txn_perf_counters[name] = (count, cum_time) + with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( inner_func, *args, **kwargs -- cgit 1.4.1 From 347b497db0355fe4e26ae3a51967aa91bec090d3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 17:57:09 +0000 Subject: Formatting --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index bcb03cbdcb..45f4b994eb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -39,7 +39,6 @@ class LoggingTransaction(object): passed to the constructor. Adds logging to the .execute() method.""" __slots__ = ["txn", "name"] - def __init__(self, txn, name): object.__setattr__(self, "txn", txn) object.__setattr__(self, "name", name) -- cgit 1.4.1 From 0c4536da8fe75a207052fb558414b4408aa857ec Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 18:06:31 +0000 Subject: Use the transaction 'desc' rather than 'name', increment the txn_ids in txn names --- synapse/storage/_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 45f4b994eb..5ddd410607 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -140,7 +140,7 @@ class SQLBaseStore(object): with LoggingContext("runInteraction") as context: current_context.copy_to(context) start = time.time() * 1000 - txn_id = SQLBaseStore._TXN_ID + txn_id = self._TXN_ID # We don't really need these to be unique, so lets stop it from # growing really large. @@ -163,10 +163,10 @@ class SQLBaseStore(object): self._current_txn_total_time += end - start - count, cum_time = self._txn_perf_counters.get(name, (0,0)) + count, cum_time = self._txn_perf_counters.get(desc, (0,0)) count += 1 cum_time += end - start - self._txn_perf_counters[name] = (count, cum_time) + self._txn_perf_counters[desc] = (count, cum_time) with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( -- cgit 1.4.1 From 8ce100c7b48a38e5f74ed6ac560de1f50c288379 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 18:29:36 +0000 Subject: Convert directory paths to absolute paths before daemonizing --- synapse/config/_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/config/_base.py b/synapse/config/_base.py index dfc115d8e8..9b0f8c3c32 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -50,8 +50,9 @@ class Config(object): ) return cls.abspath(file_path) - @staticmethod - def ensure_directory(dir_path): + @classmethod + def ensure_directory(cls, dir_path): + dir_path = cls.abspath(dir_path) if not os.path.exists(dir_path): os.makedirs(dir_path) if not os.path.isdir(dir_path): -- cgit 1.4.1 From d19e2ed02fb545bde9c8d1a317f07b9fb786d2c5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 11:01:15 +0000 Subject: Move construction of object within if block --- synapse/state.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 49ac09863e..2dea8f552a 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -306,13 +306,13 @@ class StateHandler(object): new_state = unconflicted_state new_state.update(resolved_state) - cache = _StateCacheEntry( - state=new_state, - state_group=None, - ts=self.clock.time_msec() - ) - if self._state_cache is not None: + cache = _StateCacheEntry( + state=new_state, + state_group=None, + ts=self.clock.time_msec() + ) + self._state_cache[frozenset(event_ids)] = cache defer.returnValue((None, new_state, prev_states)) -- cgit 1.4.1 From 2b042ad67fd16c43f3cfce472a8d1b31cff867da Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 11:03:16 +0000 Subject: Oops, we do want to defer.return regardless of whether we are caching or not --- synapse/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 2dea8f552a..31f503a1ee 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -261,7 +261,7 @@ class StateHandler(object): self._state_cache[frozenset(event_ids)] = cache - defer.returnValue((name, state, prev_states)) + defer.returnValue((name, state, prev_states)) state = {} for group, g_state in state_groups.items(): -- cgit 1.4.1 From 30595b466f1f6b61eef15dc41562f6336e57e6fe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 13:50:33 +0000 Subject: Use yaml logging config format because it is much nicer --- synapse/config/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/config/logger.py b/synapse/config/logger.py index f9568ebd21..b18d08ec61 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -18,6 +18,7 @@ from synapse.util.logcontext import LoggingContextFilter from twisted.python.log import PythonLoggingObserver import logging import logging.config +import yaml class LoggingConfig(Config): @@ -79,7 +80,7 @@ class LoggingConfig(Config): logger.addHandler(handler) logger.info("Test") else: - logging.config.fileConfig(self.log_config) + logging.config.dictConfig(yaml.load(self.log_config)) observer = PythonLoggingObserver() observer.start() -- cgit 1.4.1 From f91345bdb5b76ce76b99d4fc969ee4f41cbbba4c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 13:57:31 +0000 Subject: yaml.load expects strings to be a yaml rather than file --- synapse/config/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/config/logger.py b/synapse/config/logger.py index b18d08ec61..63c8e36930 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -80,7 +80,8 @@ class LoggingConfig(Config): logger.addHandler(handler) logger.info("Test") else: - logging.config.dictConfig(yaml.load(self.log_config)) + with open(self.log_config, 'r') as f: + logging.config.dictConfig(yaml.load(f)) observer = PythonLoggingObserver() observer.start() -- cgit 1.4.1 From d7c7efb691bd726ec3e8879e289546fbcfd7dabd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 14:50:53 +0000 Subject: Add performance counters for different stages of loading events --- synapse/storage/_base.py | 84 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 24 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5ddd410607..c79399fe5e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -77,6 +77,43 @@ class LoggingTransaction(object): sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) +class PerformanceCounters(object): + def __init__(self): + self.current_counters = {} + self.previous_counters = {} + + def update(self, key, start_time, end_time=None): + if end_time is None: + end_time = time.time() * 1000; + duration = end_time - start_time + count, cum_time = self.current_counters.get(key, (0, 0)) + count += 1 + cum_time += duration + self.current_counters[key] = (count, cum_time) + return end_time + + def interval(self, interval_duration, limit=3): + counters = [] + for name, (count, cum_time) in self.current_counters.items(): + prev_count, prev_time = self.previous_counters.get(name, (0, 0)) + counters.append(( + (cum_time - prev_time) / interval_duration, + count - prev_count, + name + )) + + self.previous_counters = dict(self.current_counters) + + counters.sort(reverse=True) + + top_n_counters = ", ".join( + "%s(%d): %.3f%%" % (name, count, 100 * ratio) + for ratio, count, name in txn_counters[:limit] + ) + + return top_n_counters + + class SQLBaseStore(object): _TXN_ID = 0 @@ -88,8 +125,8 @@ class SQLBaseStore(object): self._previous_txn_total_time = 0 self._current_txn_total_time = 0 self._previous_loop_ts = 0 - self._txn_perf_counters = {} - self._previous_txn_perf_counters = {} + self._txn_perf_counters = PerformanceCounters() + self._get_event_counters = PerformanceCounters() def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -105,23 +142,12 @@ class SQLBaseStore(object): ratio = (curr - prev)/(time_now - time_then) - txn_counters = [] - for name, (count, cum_time) in self._txn_perf_counters.items(): - prev_count, prev_time = self._previous_txn_perf_counters.get( - name, (0,0) - ) - txn_counters.append(( - (cum_time - prev_time) / (time_now - time_then), - count - prev_count, - name - )) - - self._previous_txn_perf_counters = dict(self._txn_perf_counters) - - txn_counters.sort(reverse=True) - top_three_counters = ", ".join( - "%s(%d): %.3f%%" % (name, count, 100 * ratio) - for ratio, count, name in txn_counters[:3] + top_three_counters = self._txn_perf_counters.interval( + time_now - time_then, limit=3 + ) + + top_3_event_counters = self._get_event_counters.interval( + time_now - time_then, limit=3 ) logger.info( @@ -162,11 +188,7 @@ class SQLBaseStore(object): ) self._current_txn_total_time += end - start - - count, cum_time = self._txn_perf_counters.get(desc, (0,0)) - count += 1 - cum_time += end - start - self._txn_perf_counters[desc] = (count, cum_time) + self._txn_perf_counters.update(desc, start, end) with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( @@ -566,6 +588,8 @@ class SQLBaseStore(object): "LIMIT 1 " ) + start_time = time.time() * 1000; + txn.execute(sql, (event_id,)) res = txn.fetchone() @@ -575,6 +599,8 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res + self._get_event_counters.update("select_event", start_time) + if allow_rejected or not rejected_reason: return self._get_event_from_row_txn( txn, internal_metadata, js, redacted, @@ -586,10 +612,18 @@ class SQLBaseStore(object): def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False): + + start_time = time.time() * 1000; + update_counter = self._get_event_counters.update + d = json.loads(js) + start_time = update_counter("decode_json", start_time) + internal_metadata = json.loads(internal_metadata) + start_time = update_counter("decode_internal", start_time) ev = FrozenEvent(d, internal_metadata_dict=internal_metadata) + start_time = update_counter("build_frozen_event", start_time) if check_redacted and redacted: ev = prune_event(ev) @@ -605,6 +639,7 @@ class SQLBaseStore(object): if because: ev.unsigned["redacted_because"] = because + start_time = update_counter("redact_event", start_time) if get_prev_content and "replaces_state" in ev.unsigned: prev = self._get_event_txn( @@ -614,6 +649,7 @@ class SQLBaseStore(object): ) if prev: ev.unsigned["prev_content"] = prev.get_dict()["content"] + start_time = update_counter("get_prev_content", start_time) return ev -- cgit 1.4.1 From fda4422bc9d9f2974d7185011d6d905eea372b09 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 14:54:07 +0000 Subject: Fix pyflakes --- synapse/storage/_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c79399fe5e..36455ef93c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -108,7 +108,7 @@ class PerformanceCounters(object): top_n_counters = ", ".join( "%s(%d): %.3f%%" % (name, count, 100 * ratio) - for ratio, count, name in txn_counters[:limit] + for ratio, count, name in counters[:limit] ) return top_n_counters @@ -151,8 +151,8 @@ class SQLBaseStore(object): ) logger.info( - "Total database time: %.3f%% {%s}", - ratio * 100, top_three_counters + "Total database time: %.3f%% {%s} {%s}", + ratio * 100, top_three_counters, top_3_event_counters ) self._clock.looping_call(loop, 10000) -- cgit 1.4.1 From f8abbae99fe76409533518e828031de4933f38ab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 15:45:50 +0000 Subject: Remove unnecessary logging --- synapse/state.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 64c58a3934..98aaa2be53 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -72,11 +72,7 @@ class StateHandler(object): self._state_cache = {} def f(): - logger.debug("Pruning") - try: - self._prune_cache() - except: - logger.exception("Prune") + self._prune_cache() self.clock.looping_call(f, 5*1000) @@ -400,7 +396,6 @@ class StateHandler(object): return sorted(events, key=key_func) def _prune_cache(self): - logger.debug("_prune_cache") logger.debug( "_prune_cache. before len: %d", len(self._state_cache.keys()) -- cgit 1.4.1 From 697ab75a34b14da6d872d153f36017a6dc6d5b99 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 15:46:24 +0000 Subject: Sign auth_chains when returned by /state/ requests --- synapse/federation/federation_server.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 4742ca9390..b23f72c7fa 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -25,6 +25,8 @@ from synapse.events import FrozenEvent from synapse.api.errors import FederationError, SynapseError +from synapse.crypto.event_signing import compute_event_signature + import logging @@ -156,6 +158,15 @@ class FederationServer(FederationBase): auth_chain = yield self.store.get_auth_chain( [pdu.event_id for pdu in pdus] ) + + for event in auth_chain: + event.signatures.update( + compute_event_signature( + event, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) else: raise NotImplementedError("Specify an event") -- cgit 1.4.1 From b085fac7353e1cd395b89f9334c8273a8e996f48 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 16:30:48 +0000 Subject: Code-style fixes --- synapse/handlers/presence.py | 4 +++- synapse/handlers/register.py | 13 ++++++++----- synapse/push/__init__.py | 8 +++++--- synapse/push/baserules.py | 11 ++++++----- synapse/push/rulekinds.py | 12 ++++++------ synapse/python_dependencies.py | 6 ++++-- synapse/rest/client/v1/push_rule.py | 18 +++++++++++++----- synapse/rest/client/v1/pusher.py | 4 ++-- synapse/rest/media/v1/upload_resource.py | 7 ++++--- synapse/storage/_base.py | 6 +++--- synapse/storage/push_rule.py | 4 +++- 11 files changed, 57 insertions(+), 36 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index cd0798c2b0..6a266ee0fe 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -658,7 +658,9 @@ class PresenceHandler(BaseHandler): observers = set(self._remote_recvmap.get(user, set())) if observers: - logger.debug(" | %d interested local observers %r", len(observers), observers) + logger.debug( + " | %d interested local observers %r", len(observers), observers + ) rm_handler = self.homeserver.get_handlers().room_member_handler room_ids = yield rm_handler.get_rooms_for_user(user) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 4f06c487b1..0247327eb9 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -105,17 +105,20 @@ class RegistrationHandler(BaseHandler): # do it here. try: auth_user = UserID.from_string(user_id) - identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) - upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + media_repository = self.hs.get_resource_for_media_repository() + identicon_resource = media_repository.getChildWithDefault("identicon", None) + upload_resource = media_repository.getChildWithDefault("upload", None) identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) content_uri = yield upload_resource.create_content( "image/png", None, identicon_bytes, len(identicon_bytes), auth_user ) profile_handler = self.hs.get_handlers().profile_handler - profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + profile_handler.set_avatar_url( + auth_user, auth_user, ("%s#auto" % (content_uri,)) + ) except NotImplementedError: - pass # make tests pass without messing around creating default avatars - + pass # make tests pass without messing around creating default avatars + defer.returnValue((user_id, token)) @defer.inlineCallbacks diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 6f143a5df9..418a348a58 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -140,7 +140,7 @@ class Pusher(object): lambda x: ('[%s%s]' % (x.group(1) and '^' or '', re.sub(r'\\\-', '-', x.group(2)))), r) return r - + def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): if condition['kind'] == 'event_match': if 'pattern' not in condition: @@ -170,8 +170,10 @@ class Pusher(object): return False if not display_name: return False - return re.search("\b%s\b" % re.escape(display_name), - ev['content']['body'], flags=re.IGNORECASE) is not None + return re.search( + "\b%s\b" % re.escape(display_name), ev['content']['body'], + flags=re.IGNORECASE + ) is not None elif condition['kind'] == 'room_member_count': if 'is' not in condition: diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 37878f1e0b..162d265f66 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -1,5 +1,6 @@ from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP + def list_with_base_rules(rawrules, user_name): ruleslist = [] @@ -9,9 +10,9 @@ def list_with_base_rules(rawrules, user_name): if r['priority_class'] < current_prio_class: while r['priority_class'] < current_prio_class: ruleslist.extend(make_base_rules( - user_name, - PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) - ) + user_name, + PRIORITY_CLASS_INVERSE_MAP[current_prio_class] + )) current_prio_class -= 1 ruleslist.append(r) @@ -19,8 +20,8 @@ def list_with_base_rules(rawrules, user_name): while current_prio_class > 0: ruleslist.extend(make_base_rules( user_name, - PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) - ) + PRIORITY_CLASS_INVERSE_MAP[current_prio_class] + )) current_prio_class -= 1 return ruleslist diff --git a/synapse/push/rulekinds.py b/synapse/push/rulekinds.py index 763bdee58e..660aa4e10e 100644 --- a/synapse/push/rulekinds.py +++ b/synapse/push/rulekinds.py @@ -1,8 +1,8 @@ PRIORITY_CLASS_MAP = { - 'underride': 1, - 'sender': 2, - 'room': 3, - 'content': 4, - 'override': 5, - } + 'underride': 1, + 'sender': 2, + 'room': 3, + 'content': 4, + 'override': 5, +} PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a89d618606..fd68da9dfb 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -19,10 +19,11 @@ REQUIREMENTS = { "pydenticon": ["pydenticon"], } + def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) -DEPENDENCY_LINKS=[ +DEPENDENCY_LINKS = [ github_link( project="matrix-org/syutil", version="v0.0.2", @@ -101,6 +102,7 @@ def check_requirements(): % (dependency, file_path, version, required_version) ) + def list_requirements(): result = [] linked = [] @@ -111,7 +113,7 @@ def list_requirements(): for requirement in REQUIREMENTS: is_linked = False for link in linked: - if requirement.replace('-','_').startswith(link): + if requirement.replace('-', '_').startswith(link): is_linked = True if not is_linked: result.append(requirement) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index d43ade39dd..c4e7dfcf0e 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -15,12 +15,17 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError, \ - StoreError +from synapse.api.errors import ( + SynapseError, Codes, UnrecognizedRequestError, NotFoundError, StoreError +) from .base import ClientV1RestServlet, client_path_pattern -from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException +from synapse.storage.push_rule import ( + InconsistentRuleException, RuleNotFoundException +) import synapse.push.baserules as baserules -from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP +from synapse.push.rulekinds import ( + PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP +) import json @@ -105,7 +110,9 @@ class PushRuleRestServlet(ClientV1RestServlet): # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference - rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name( + user.to_string() + ) for r in rawrules: r["conditions"] = json.loads(r["conditions"]) @@ -383,6 +390,7 @@ def _namespaced_rule_id_from_spec(spec): def _rule_id_from_namespaced(in_rule_id): return in_rule_id.split('/')[-1] + class InvalidRuleException(Exception): pass diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index e10d2576d2..80e9939b79 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -34,8 +34,8 @@ class PusherRestServlet(ClientV1RestServlet): pusher_pool = self.hs.get_pusherpool() if ('pushkey' in content and 'app_id' in content - and 'kind' in content and - content['kind'] is None): + and 'kind' in content and + content['kind'] is None): yield pusher_pool.remove_pusher( content['app_id'], content['pushkey'] ) diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 5b42782331..6df52ca434 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -38,9 +38,10 @@ class UploadResource(BaseMediaResource): def render_OPTIONS(self, request): respond_with_json(request, 200, {}, send_cors=True) return NOT_DONE_YET - + @defer.inlineCallbacks - def create_content(self, media_type, upload_name, content, content_length, auth_user): + def create_content(self, media_type, upload_name, content, content_length, + auth_user): media_id = random_string(24) fname = self.filepaths.local_media_filepath(media_id) @@ -65,7 +66,7 @@ class UploadResource(BaseMediaResource): } yield self._generate_local_thumbnails(media_id, media_info) - + defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) @defer.inlineCallbacks diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 36455ef93c..3e1ab0a159 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -84,7 +84,7 @@ class PerformanceCounters(object): def update(self, key, start_time, end_time=None): if end_time is None: - end_time = time.time() * 1000; + end_time = time.time() * 1000 duration = end_time - start_time count, cum_time = self.current_counters.get(key, (0, 0)) count += 1 @@ -588,7 +588,7 @@ class SQLBaseStore(object): "LIMIT 1 " ) - start_time = time.time() * 1000; + start_time = time.time() * 1000 txn.execute(sql, (event_id,)) @@ -613,7 +613,7 @@ class SQLBaseStore(object): def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False): - start_time = time.time() * 1000; + start_time = time.time() * 1000 update_counter = self._get_event_counters.update d = json.loads(js) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 30e23445d9..620de71398 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -91,7 +91,9 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, relative_to_rule)) res = txn.fetchall() if not res: - raise RuleNotFoundException("before/after rule not found: %s" % (relative_to_rule)) + raise RuleNotFoundException( + "before/after rule not found: %s" % (relative_to_rule,) + ) priority_class, base_rule_priority = res[0] if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: -- cgit 1.4.1 From e8d4a314753bb49f91a2e51d0e442a153f519a64 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 16:36:51 +0000 Subject: Fix prune_events to work with nested dicts --- synapse/events/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 4ad37188ba..5bfa820f96 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -45,12 +45,14 @@ def prune_event(event): "membership", ] + event_dict = event.get_dict() + new_content = {} def add_fields(*fields): for field in fields: if field in event.content: - new_content[field] = event.content[field] + new_content[field] = event_dict["content"][field] if event_type == EventTypes.Member: add_fields("membership") @@ -75,7 +77,7 @@ def prune_event(event): allowed_fields = { k: v - for k, v in event.get_dict().items() + for k, v in event_dict.items() if k in allowed_keys } -- cgit 1.4.1 From b61a308b2725c6e7ab12983d2d05461a532f6405 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 16:37:12 +0000 Subject: Bump version --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/VERSION b/VERSION index 1648b8029d..f97571ce52 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1e +0.6.1f diff --git a/synapse/__init__.py b/synapse/__init__.py index d3f1c33d3a..329138734a 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.1e" +__version__ = "0.6.1f" -- cgit 1.4.1 From c8e1da930dd4ffbc4ca409208b2d715182cd100f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 17:30:46 +0000 Subject: Log all the exits from _attempt_new_transaction --- synapse/federation/transaction_queue.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index f38aeba7cc..731019ad9f 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -157,15 +157,19 @@ class TransactionQueue(object): else: logger.info("TX [%s] is ready for retry", destination) - logger.info("TX [%s] _attempt_new_transaction", destination) - if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending # request at which point pending_pdus_by_dest just keeps growing. # we need application-layer timeouts of some flavour of these # requests + logger.info( + "TX [%s] Transaction already in progress", + destination + ) return + logger.info("TX [%s] _attempt_new_transaction", destination) + # list of (pending_pdu, deferred, order) pending_pdus = self.pending_pdus_by_dest.pop(destination, []) pending_edus = self.pending_edus_by_dest.pop(destination, []) @@ -176,6 +180,7 @@ class TransactionQueue(object): destination, len(pending_pdus)) if not pending_pdus and not pending_edus and not pending_failures: + logger.info("TX [%s] Nothing to send", destination) return logger.debug( -- cgit 1.4.1 From 7ed971d9b206a70f44c55588bdb1a4a33906fbe1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 17:42:36 +0000 Subject: Single source version and python dependencies, prevent people accidentally installing with easy_install, use scripts rather than entry_points to install synctl --- VERSION | 1 - setup.py | 80 +++++++++++++++++++++--------------------- synapse/python_dependencies.py | 6 ++-- 3 files changed, 43 insertions(+), 44 deletions(-) delete mode 100644 VERSION (limited to 'synapse') diff --git a/VERSION b/VERSION deleted file mode 100644 index 1c29ff4d36..0000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.6.1d diff --git a/setup.py b/setup.py index bd2766b24c..0215fba4c4 100755 --- a/setup.py +++ b/setup.py @@ -17,52 +17,52 @@ import os from setuptools import setup, find_packages +import setuptools.command.easy_install -# Utility function to read the README file. -# Used for the long_description. It's nice, because now 1) we have a top level -# README file and 2) it's easier to type in the README file than to put a raw -# string in below ... -def read(fname): - return open(os.path.join(os.path.dirname(__file__), fname)).read() + +def no_easy_install(self, spec, *args, **kargs): + raise RuntimeError( + "Missing requirement %r. Please use pip to install this dependency." + " See README.rst for instructions." + % (spec,) + ) + +# Patch the easy_install command to prevent people accidentally installing +# depedencies using it. +setuptools.command.easy_install.easy_install.easy_install = no_easy_install + + +here = os.path.abspath(os.path.dirname(__file__)) + + +def read_file(path): + """Read a file from the package. Takes a list of strings to join to + make the path""" + file_path = os.path.join(here, *path) + with open(file_path) as f: + return f.read() + + +def exec_file(path): + """Execute a single python file to get the variables defined in it""" + result = {} + code = read_file(path) + exec(code, result) + return result + +version = exec_file(("synapse", "__init__.py"))["__version__"] +dependencies = exec_file(("synapse", "python_dependencies.py")) +long_description = read_file(("README.rst",)) setup( name="matrix-synapse", - version=read("VERSION").strip(), + version=version, packages=find_packages(exclude=["tests", "tests.*"]), description="Reference Synapse Home Server", - install_requires=[ - "syutil==0.0.2", - "matrix_angular_sdk>=0.6.1", - "Twisted==14.0.2", - "service_identity>=1.0.0", - "pyopenssl>=0.14", - "pyyaml", - "pyasn1", - "pynacl", - "daemonize", - "py-bcrypt", - "frozendict>=0.4", - "pillow", - "pydenticon", - ], - dependency_links=[ - "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", - "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.1/#egg=matrix_angular_sdk-0.6.1", - ], - setup_requires=[ - "Twisted==14.0.2", # Here to override setuptools_trial's dependency on Twisted>=2.4.0 - "setuptools_trial", - "setuptools>=1.0.0", # Needs setuptools that supports git+ssh. - # TODO: Do we need this now? we don't use git+ssh. - "mock" - ], + install_requires=dependencies["REQUIREMENTS"].keys(), + dependency_links=dependencies["DEPENDENCY_LINKS"], include_package_data=True, zip_safe=False, - long_description=read("README.rst"), - entry_points=""" - [console_scripts] - synctl=synapse.app.synctl:main - synapse-homeserver=synapse.app.homeserver:main - """ + long_description=long_description, + scripts=["synctl"], ) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index fd68da9dfb..e2a9d1f6a7 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], - "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], + "matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -31,8 +31,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/matrix-angular-sdk", - version="v0.6.0", - egg="matrix_angular_sdk-0.6.0", + version="v0.6.1", + egg="matrix_angular_sdk-0.6.1", ), github_link( project="pyca/pynacl", -- cgit 1.4.1 From 84a769cdb7321481954c386a040d7e7ff504d02b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 17:58:36 +0000 Subject: Fix code-style --- synapse/app/homeserver.py | 2 +- synapse/crypto/keyclient.py | 4 ++-- synapse/handlers/presence.py | 8 ++++---- synapse/handlers/sync.py | 2 +- synapse/rest/client/v1/directory.py | 2 +- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/media/v1/upload_resource.py | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8976ff2e82..ff6680e7da 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -134,7 +134,7 @@ class SynapseHomeServer(HomeServer): logger.info("Attaching %s to path %s", resource, full_path) last_resource = self.root_resource for path_seg in full_path.split('/')[1:-1]: - if not path_seg in last_resource.listNames(): + if path_seg not in last_resource.listNames(): # resource doesn't exist, so make a "dummy resource" child_resource = Resource() last_resource.putChild(path_seg, child_resource) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index cdb6279764..cd12349f67 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -75,7 +75,7 @@ class SynapseKeyClientProtocol(HTTPClient): def handleStatus(self, version, status, message): if status != b"200": - #logger.info("Non-200 response from %s: %s %s", + # logger.info("Non-200 response from %s: %s %s", # self.transport.getHost(), status, message) self.transport.abortConnection() @@ -83,7 +83,7 @@ class SynapseKeyClientProtocol(HTTPClient): try: json_response = json.loads(response_body_bytes) except ValueError: - #logger.info("Invalid JSON response from %s", + # logger.info("Invalid JSON response from %s", # self.transport.getHost()) self.transport.abortConnection() return diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6a266ee0fe..59287010ed 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -457,9 +457,9 @@ class PresenceHandler(BaseHandler): if state is None: state = yield self.store.get_presence_state(user.localpart) else: -# statuscache = self._get_or_make_usercache(user) -# self._user_cachemap_latest_serial += 1 -# statuscache.update(state, self._user_cachemap_latest_serial) + # statuscache = self._get_or_make_usercache(user) + # self._user_cachemap_latest_serial += 1 + # statuscache.update(state, self._user_cachemap_latest_serial) pass yield self.push_update_to_local_and_remote( @@ -709,7 +709,7 @@ class PresenceHandler(BaseHandler): # TODO(paul) permissions checks - if not user in self._remote_sendmap: + if user not in self._remote_sendmap: self._remote_sendmap[user] = set() self._remote_sendmap[user].add(origin) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5af90cc5d1..7883bbd834 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -114,7 +114,7 @@ class SyncHandler(BaseHandler): if sync_config.gap: return self.incremental_sync_with_gap(sync_config, since_token) else: - #TODO(mjark): Handle gapless sync + # TODO(mjark): Handle gapless sync raise NotImplementedError() @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 8f65efec5f..8ed7e2d669 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -48,7 +48,7 @@ class ClientDirectoryServer(ClientV1RestServlet): user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) - if not "room_id" in content: + if "room_id" not in content: raise SynapseError(400, "Missing room_id key", errcode=Codes.BAD_JSON) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 81d5cf8ead..3056ec45cf 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -118,7 +118,7 @@ class SyncRestServlet(RestServlet): except: filter = Filter({}) # filter = filter.apply_overrides(http_request) - #if filter.matches(event): + # if filter.matches(event): # # stuff sync_config = SyncConfig( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 6df52ca434..e5aba3af4c 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -96,8 +96,8 @@ class UploadResource(BaseMediaResource): code=400, ) - #if headers.hasHeader("Content-Disposition"): - # disposition = headers.getRawHeaders("Content-Disposition")[0] + # if headers.hasHeader("Content-Disposition"): + # disposition = headers.getRawHeaders("Content-Disposition")[0] # TODO(markjh): parse content-dispostion content_uri = yield self.create_content( -- cgit 1.4.1 From 0e6b3e4e40ae918eacdef87bb50ff1d19b304e7c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 18:17:27 +0000 Subject: Time out HTTP federation requests --- synapse/http/matrixfederationclient.py | 11 +++++++++-- synapse/util/async.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index aa14782b0f..fdc1e2678e 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -22,7 +22,7 @@ from twisted.web._newclient import ResponseDone from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint -from synapse.util.async import sleep +from synapse.util.async import sleep, time_bound_deferred from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json @@ -78,6 +78,7 @@ class MatrixFederationHttpClient(object): self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname self.agent = MatrixFederationHttpAgent(reactor) + self.clock = hs.get_clock() @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, @@ -117,7 +118,7 @@ class MatrixFederationHttpClient(object): try: with PreserveLoggingContext(): - response = yield self.agent.request( + request_deferred = self.agent.request( destination, endpoint, method, @@ -128,6 +129,12 @@ class MatrixFederationHttpClient(object): producer ) + response = yield time_bound_deferred( + request_deferred, + clock=self.clock, + time_out=60, + ) + logger.debug("Got response to %s", method) break except Exception as e: diff --git a/synapse/util/async.py b/synapse/util/async.py index c4fe5d522f..d4d1d4b472 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -32,3 +32,23 @@ def run_on_reactor(): iteration of the main loop """ return sleep(0) + + +def time_bound_deferred(given_deferred, clock, time_out): + ret_deferred = defer.Deferred() + + def timed_out(): + if not given_deferred.called: + given_deferred.cancel() + ret_deferred.errback(RuntimeError("Timed out")) + + timer = clock.call_later(time_out, timed_out) + + def succeed(result): + clock.cancel_call_later(timer) + ret_deferred.callback(result) + + given_deferred.addCallback(succeed) + given_deferred.addErrback(ret_deferred.errback) + + return ret_deferred -- cgit 1.4.1 From dcf52469e821b3b2b69a0610c2c4f025a5aac68f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 10:25:06 +0000 Subject: Move time_bound_deferred into Clock --- synapse/http/matrixfederationclient.py | 5 ++--- synapse/util/__init__.py | 21 ++++++++++++++++++++- synapse/util/async.py | 20 -------------------- 3 files changed, 22 insertions(+), 24 deletions(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index fdc1e2678e..74e523960f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -22,7 +22,7 @@ from twisted.web._newclient import ResponseDone from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint -from synapse.util.async import sleep, time_bound_deferred +from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json @@ -129,9 +129,8 @@ class MatrixFederationHttpClient(object): producer ) - response = yield time_bound_deferred( + response = yield self.clock.time_bound_deferred( request_deferred, - clock=self.clock, time_out=60, ) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 4e837a918e..2da8dfa719 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -15,7 +15,7 @@ from synapse.util.logcontext import LoggingContext -from twisted.internet import reactor +from twisted.internet import defer, reactor import time @@ -45,3 +45,22 @@ class Clock(object): def cancel_call_later(self, timer): timer.cancel() + + def time_bound_deferred(self, given_deferred, time_out): + ret_deferred = defer.Deferred() + + def timed_out(): + if not given_deferred.called: + given_deferred.cancel() + ret_deferred.errback(RuntimeError("Timed out")) + + timer = self.call_later(time_out, timed_out) + + def succeed(result): + self.cancel_call_later(timer) + ret_deferred.callback(result) + + given_deferred.addCallback(succeed) + given_deferred.addErrback(ret_deferred.errback) + + return ret_deferred diff --git a/synapse/util/async.py b/synapse/util/async.py index d4d1d4b472..c4fe5d522f 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -32,23 +32,3 @@ def run_on_reactor(): iteration of the main loop """ return sleep(0) - - -def time_bound_deferred(given_deferred, clock, time_out): - ret_deferred = defer.Deferred() - - def timed_out(): - if not given_deferred.called: - given_deferred.cancel() - ret_deferred.errback(RuntimeError("Timed out")) - - timer = clock.call_later(time_out, timed_out) - - def succeed(result): - clock.cancel_call_later(timer) - ret_deferred.callback(result) - - given_deferred.addCallback(succeed) - given_deferred.addErrback(ret_deferred.errback) - - return ret_deferred -- cgit 1.4.1 From 05b961d7e3f6c926520c3200ae3031c77cebc212 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 10:28:46 +0000 Subject: PEP8 --- synapse/http/matrixfederationclient.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 74e523960f..92f887f778 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -72,7 +72,6 @@ class MatrixFederationHttpClient(object): requests. """ - def __init__(self, hs): self.hs = hs self.signing_key = hs.config.signing_key[0] -- cgit 1.4.1 From 6370cffbbfc0bcfa4465453c24c535ef755db4f1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 10:34:41 +0000 Subject: Fix bug where variable was not always defined --- synapse/http/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/http/server.py b/synapse/http/server.py index 6d084fa33c..589c30c199 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -113,6 +113,7 @@ class JsonResource(HttpServer, resource.Resource): path. """ code = None + start = self.clock.time_msec() try: # Just say yes to OPTIONS. if request.method == "OPTIONS": @@ -137,8 +138,6 @@ class JsonResource(HttpServer, resource.Resource): request.method, request.path ) - start = self.clock.time_msec() - code, response = yield path_entry.callback( request, *args -- cgit 1.4.1 From c7783d6feec9b69c24f3303cbb51cce3e6b8ffb3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:36:08 +0000 Subject: Notify ASes for events sent by other users in a room which an AS user is a part of. --- synapse/appservice/__init__.py | 17 +++++++++++++---- synapse/handlers/appservice.py | 21 ++++++++++++++++----- synapse/rest/appservice/v1/__init__.py | 4 ++-- tests/appservice/test_appservice.py | 25 +++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index fb9bfffe5d..381b4cfc4a 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -74,7 +74,7 @@ class ApplicationService(object): return True return False - def _matches_user(self, event): + def _matches_user(self, event, member_list): if (hasattr(event, "sender") and self.is_interested_in_user(event.sender)): return True @@ -83,6 +83,10 @@ class ApplicationService(object): and hasattr(event, "state_key") and self.is_interested_in_user(event.state_key)): return True + # check joined member events + for member in member_list: + if self.is_interested_in_user(member.state_key): + return True return False def _matches_room_id(self, event): @@ -96,7 +100,8 @@ class ApplicationService(object): return True return False - def is_interested(self, event, restrict_to=None, aliases_for_event=None): + def is_interested(self, event, restrict_to=None, aliases_for_event=None, + member_list=None): """Check if this service is interested in this event. Args: @@ -104,18 +109,22 @@ class ApplicationService(object): restrict_to(str): The namespace to restrict regex tests to. aliases_for_event(list): A list of all the known room aliases for this event. + member_list(list): A list of all joined room members in this room. Returns: bool: True if this service would like to know about this event. """ if aliases_for_event is None: aliases_for_event = [] + if member_list is None: + member_list = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: # this is a programming error, so fail early and raise a general # exception raise Exception("Unexpected restrict_to value: %s". restrict_to) if not restrict_to: - return (self._matches_user(event) + return (self._matches_user(event, member_list) or self._matches_aliases(event, aliases_for_event) or self._matches_room_id(event)) elif restrict_to == ApplicationService.NS_ALIASES: @@ -123,7 +132,7 @@ class ApplicationService(object): elif restrict_to == ApplicationService.NS_ROOMS: return self._matches_room_id(event) elif restrict_to == ApplicationService.NS_USERS: - return self._matches_user(event) + return self._matches_user(event, member_list) def is_interested_in_user(self, user_id): return self._matches_regex(user_id, ApplicationService.NS_USERS) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8591a77bf3..2c488a46f6 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, Membership from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.types import UserID @@ -154,14 +154,25 @@ class ApplicationServicesHandler(object): list: A list of services interested in this event based on the service regex. """ - # We need to know the aliases associated with this event.room_id, if any - if not alias_list: - alias_list = yield self.store.get_aliases_for_room(event.room_id) + member_list = None + if hasattr(event, "room_id"): + # We need to know the aliases associated with this event.room_id, + # if any. + if not alias_list: + alias_list = yield self.store.get_aliases_for_room( + event.room_id + ) + # We need to know the members associated with this event.room_id, + # if any. + member_list = yield self.store.get_room_members( + room_id=event.room_id, + membership=Membership.JOIN + ) services = yield self.store.get_app_services() interested_list = [ s for s in services if ( - s.is_interested(event, restrict_to, alias_list) + s.is_interested(event, restrict_to, alias_list, member_list) ) ] defer.returnValue(interested_list) diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py index bf243b6180..a7877609ad 100644 --- a/synapse/rest/appservice/v1/__init__.py +++ b/synapse/rest/appservice/v1/__init__.py @@ -21,9 +21,9 @@ class AppServiceRestResource(JsonResource): """A resource for version 1 of the matrix application service API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod def register_servlets(appservice_resource, hs): - register.register_servlets(hs, appservice_resource) \ No newline at end of file + register.register_servlets(hs, appservice_resource) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index c0aaf12785..d12e4f2644 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -143,3 +143,28 @@ class ApplicationServiceTestCase(unittest.TestCase): restrict_to=ApplicationService.NS_USERS, aliases_for_event=["#xmpp_barfoo:matrix.org"] )) + + def test_member_list_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + join_list = [ + Mock( + type="m.room.member", room_id="!foo:bar", sender="@alice:here", + state_key="@alice:here" + ), + Mock( + type="m.room.member", room_id="!foo:bar", sender="@irc_fo:here", + state_key="@irc_fo:here" # AS user + ), + Mock( + type="m.room.member", room_id="!foo:bar", sender="@bob:here", + state_key="@bob:here" + ) + ] + + self.event.sender = "@xmpp_foobar:matrix.org" + self.assertTrue(self.service.is_interested( + event=self.event, + member_list=join_list + )) -- cgit 1.4.1 From 8beb61391609d8dd55022a14e1ef11bf353c1fdc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:36:48 +0000 Subject: Add newline to EOF --- synapse/rest/appservice/v1/__init__.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py index a7877609ad..a9d6d72566 100644 --- a/synapse/rest/appservice/v1/__init__.py +++ b/synapse/rest/appservice/v1/__init__.py @@ -27,3 +27,4 @@ class AppServiceRestResource(JsonResource): @staticmethod def register_servlets(appservice_resource, hs): register.register_servlets(hs, appservice_resource) + -- cgit 1.4.1 From fd40d992adfb8b63f6e925dad030c63498501408 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:41:33 +0000 Subject: PEP8-ify --- synapse/appservice/api.py | 2 -- synapse/handlers/directory.py | 3 --- synapse/rest/appservice/v1/register.py | 2 +- synapse/storage/appservice.py | 3 --- 4 files changed, 1 insertion(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 15ac1e27fc..6192813c03 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -71,7 +71,6 @@ class ApplicationServiceApi(SimpleHttpClient): logger.warning("query_alias to %s threw exception %s", uri, ex) defer.returnValue(False) - @defer.inlineCallbacks def push_bulk(self, service, events): events = self._serialize(events) @@ -107,4 +106,3 @@ class ApplicationServiceApi(SimpleHttpClient): return [ serialize_event(e, time_now, as_client_event=True) for e in events ] - diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 87bc12c983..20ab9e269c 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -58,7 +58,6 @@ class DirectoryHandler(BaseHandler): servers ) - @defer.inlineCallbacks def create_association(self, user_id, room_alias, room_id, servers=None): # association creation for human users @@ -75,7 +74,6 @@ class DirectoryHandler(BaseHandler): ) yield self._create_association(room_alias, room_id, servers) - @defer.inlineCallbacks def create_appservice_association(self, service, room_alias, room_id, servers=None): @@ -127,7 +125,6 @@ class DirectoryHandler(BaseHandler): # if room_id: # yield self._update_room_alias_events(user_id, room_id) - @defer.inlineCallbacks def get_association(self, room_alias): room_id = None diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index d3d5aef220..3bd0c1220c 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -65,7 +65,7 @@ class RegisterRestServlet(AppServiceRestServlet): hs_token = app_service.hs_token defer.returnValue((200, { - "hs_token": hs_token + "hs_token": hs_token })) def _parse_namespace(self, target_ns, origin_ns, ns): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index ba31c68595..d941b1f387 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -97,7 +97,6 @@ class ApplicationServiceStore(SQLBaseStore): # allocate new ASes. It relies on the server admin inserting the AS # token into the database manually. - if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") @@ -186,7 +185,6 @@ class ApplicationServiceStore(SQLBaseStore): # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. - @defer.inlineCallbacks def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" @@ -244,4 +242,3 @@ class ApplicationServiceStore(SQLBaseStore): hs_token=service["hs_token"], sender=service["sender"] )) - -- cgit 1.4.1 From 14d413752bc7f656e9df5363a8aa29bae404d10a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:53:47 +0000 Subject: Fix newline on __init__ --- synapse/rest/appservice/v1/__init__.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py index a9d6d72566..a7877609ad 100644 --- a/synapse/rest/appservice/v1/__init__.py +++ b/synapse/rest/appservice/v1/__init__.py @@ -27,4 +27,3 @@ class AppServiceRestResource(JsonResource): @staticmethod def register_servlets(appservice_resource, hs): register.register_servlets(hs, appservice_resource) - -- cgit 1.4.1 From 4ebbaf0d4382813ba896f3e8101de12e112cbed5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 14:23:10 +0000 Subject: Blunty replace json with simplejson --- synapse/crypto/keyclient.py | 2 +- synapse/federation/persistence.py | 2 +- synapse/federation/transport/server.py | 2 +- synapse/http/client.py | 2 +- synapse/http/matrixfederationclient.py | 2 +- synapse/push/__init__.py | 2 +- synapse/push/pusherpool.py | 2 +- synapse/rest/client/v1/directory.py | 2 +- synapse/rest/client/v1/login.py | 2 +- synapse/rest/client/v1/presence.py | 2 +- synapse/rest/client/v1/profile.py | 2 +- synapse/rest/client/v1/push_rule.py | 2 +- synapse/rest/client/v1/pusher.py | 2 +- synapse/rest/client/v1/register.py | 2 +- synapse/rest/client/v1/room.py | 2 +- synapse/rest/client/v2_alpha/filter.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/storage/__init__.py | 2 +- synapse/storage/_base.py | 2 +- synapse/storage/filtering.py | 2 +- synapse/storage/push_rule.py | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) (limited to 'synapse') diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index cd12349f67..74008347c3 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -19,7 +19,7 @@ from twisted.internet.protocol import Factory from twisted.internet import defer, reactor from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.logcontext import PreserveLoggingContext -import json +import simplejson as json import logging diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 85c82a4623..8a1afc0ca5 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -23,7 +23,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function -import json +import simplejson as json import logging diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 9c9f8d525b..2ffb37aa18 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -20,7 +20,7 @@ from synapse.api.errors import Codes, SynapseError from synapse.util.logutils import log_function import logging -import json +import simplejson as json import re diff --git a/synapse/http/client.py b/synapse/http/client.py index 198f575cfa..d500e19c81 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -23,7 +23,7 @@ from twisted.web.http_headers import Headers from StringIO import StringIO -import json +import simplejson as json import logging import urllib diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 056d446e42..406203acf2 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -33,7 +33,7 @@ from synapse.api.errors import ( from syutil.crypto.jsonsign import sign_json -import json +import simplejson as json import logging import urllib import urlparse diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 418a348a58..0659a1cb9b 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -22,7 +22,7 @@ import synapse.util.async import baserules import logging -import json +import simplejson as json import re logger = logging.getLogger(__name__) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 5a525befd7..7483d257bf 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -20,7 +20,7 @@ from httppusher import HttpPusher from synapse.push import PusherConfigException import logging -import json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 8ed7e2d669..420aa89f38 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -20,7 +20,7 @@ from synapse.api.errors import AuthError, SynapseError, Codes from synapse.types import RoomAlias from .base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json import logging diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 7116ac98e8..b2257b749d 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError from synapse.types import UserID from base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json class LoginRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 7feb4aadb1..78d4f2b128 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -21,7 +21,7 @@ from synapse.api.errors import SynapseError from synapse.types import UserID from .base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json import logging logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 15d6f3fc6c..1e77eb49cf 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -19,7 +19,7 @@ from twisted.internet import defer from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID -import json +import simplejson as json class ProfileDisplaynameRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index c4e7dfcf0e..b012f31084 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -27,7 +27,7 @@ from synapse.push.rulekinds import ( PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP ) -import json +import simplejson as json class PushRuleRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 80e9939b79..6045e86f34 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError, Codes from synapse.push import PusherConfigException from .base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json class PusherRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index c0423c2d45..d3399c446b 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -25,7 +25,7 @@ from synapse.util.async import run_on_reactor from hashlib import sha1 import hmac -import json +import simplejson as json import logging import urllib diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 410f19ccf6..0346afb1b4 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -23,7 +23,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID, RoomAlias from synapse.events.utils import serialize_event -import json +import simplejson as json import logging import urllib diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 6ddc495d23..703250cea8 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -21,7 +21,7 @@ from synapse.types import UserID from ._base import client_v2_pattern -import json +import simplejson as json import logging diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index 22e26e3cd5..e77a20fb2e 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -25,7 +25,7 @@ from twisted.web import server, resource from twisted.internet import defer import base64 -import json +import simplejson as json import logging import os import re diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a63c59a8a2..924ea89035 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -44,7 +44,7 @@ from syutil.jsonutil import encode_canonical_json from synapse.crypto.event_signing import compute_event_reference_hash -import json +import simplejson as json import logging import os diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 3e1ab0a159..b74e74ac91 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -23,7 +23,7 @@ from synapse.util.logcontext import PreserveLoggingContext, LoggingContext from twisted.internet import defer import collections -import json +import simplejson as json import sys import time diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index e86eeced45..457a11fd02 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import SQLBaseStore -import json +import simplejson as json class FilteringStore(SQLBaseStore): diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 620de71398..ae46b39cc1 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -20,7 +20,7 @@ from twisted.internet import defer import logging import copy -import json +import simplejson as json logger = logging.getLogger(__name__) -- cgit 1.4.1 From d8324d5a2b9ffc6f3a426ecd240f6c2460630025 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 14:52:23 +0000 Subject: Add a lru cache class --- synapse/util/lrucache.py | 110 ++++++++++++++++++++++++++++++++++++++++++++ tests/util/test_lrucache.py | 56 ++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 synapse/util/lrucache.py create mode 100644 tests/util/test_lrucache.py (limited to 'synapse') diff --git a/synapse/util/lrucache.py b/synapse/util/lrucache.py new file mode 100644 index 0000000000..a45c673d32 --- /dev/null +++ b/synapse/util/lrucache.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class LruCache(object): + """Least-recently-used cache.""" + # TODO(mjark) Add hit/miss counters + # TODO(mjark) Add mutex for linked list for thread safety. + def __init__(self, max_size): + cache = {} + list_root = [] + list_root[:] = [list_root, list_root, None, None] + + PREV, NEXT, KEY, VALUE = 0, 1, 2, 3 + + def add_node(key, value): + prev_node = list_root + next_node = prev_node[NEXT] + node = [prev_node, next_node, key, value] + prev_node[NEXT] = node + next_node[PREV] = node + cache[key] = node + + def move_node_to_front(node): + prev_node = node[PREV] + next_node = node[NEXT] + prev_node[NEXT] = next_node + next_node[PREV] = prev_node + prev_node = list_root + next_node = prev_node[NEXT] + node[PREV] = prev_node + node[NEXT] = next_node + prev_node[NEXT] = node + next_node[PREV] = node + + def delete_node(node): + prev_node = node[PREV] + next_node = node[NEXT] + prev_node[NEXT] = next_node + next_node[PREV] = prev_node + cache.pop(node[KEY], None) + + def cache_get(key, default=None): + node = cache.get(key, None) + if node is not None: + move_node_to_front(node) + return node[VALUE] + else: + return default + + def cache_set(key, value): + node = cache.get(key, None) + if node is not None: + move_node_to_front(node) + node[VALUE] = value + else: + add_node(key, value) + if len(cache) > max_size: + delete_node(list_root[PREV]) + + def cache_set_default(key, value): + node = cache.get(key, None) + if node is not None: + return node[VALUE] + else: + add_node(key, value) + if len(cache) > max_size: + delete_node(list_root[PREV]) + return value + + def cache_pop(key, default=None): + node = cache.get(key, None) + if node: + delete_node(node) + return node[VALUE] + else: + return default + + self.sentinel = object() + self.get = cache_get + self.set = cache_set + self.setdefault = cache_set_default + self.pop = cache_pop + + def __getitem__(self, key): + result = self.get(key, self.sentinel) + if result is self.sentinel: + raise KeyError() + else: + return result + + def __setitem__(self, key, value): + self.set(key, value) + + def __delitem__(self, key, value): + result = self.pop(key, self.sentinel) + if result is self.sentinel: + raise KeyError() diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py new file mode 100644 index 0000000000..ab934bf928 --- /dev/null +++ b/tests/util/test_lrucache.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import unittest + +from synapse.util.lrucache import LruCache + +class LruCacheTestCase(unittest.TestCase): + + def test_get_set(self): + cache = LruCache(1) + cache["key"] = "value" + self.assertEquals(cache.get("key"), "value") + self.assertEquals(cache["key"], "value") + + def test_eviction(self): + cache = LruCache(2) + cache[1] = 1 + cache[2] = 2 + + self.assertEquals(cache.get(1), 1) + self.assertEquals(cache.get(2), 2) + + cache[3] = 3 + + self.assertEquals(cache.get(1), None) + self.assertEquals(cache.get(2), 2) + self.assertEquals(cache.get(3), 3) + + def test_setdefault(self): + cache = LruCache(1) + self.assertEquals(cache.setdefault("key", 1), 1) + self.assertEquals(cache.get("key"), 1) + self.assertEquals(cache.setdefault("key", 2), 1) + self.assertEquals(cache.get("key"), 1) + + def test_pop(self): + cache = LruCache(1) + cache["key"] = 1 + self.assertEquals(cache.pop("key"), 1) + self.assertEquals(cache.pop("key"), None) + + -- cgit 1.4.1 From f5a70e0d2e890adea53b3f6565a3bbe92512a506 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 15:01:15 +0000 Subject: Add a cache for get_event --- synapse/config/_base.py | 10 ++++++++++ synapse/config/database.py | 5 +++++ synapse/storage/__init__.py | 3 +++ synapse/storage/_base.py | 24 +++++++++++++++++++++--- tests/storage/test_base.py | 5 +++-- tests/utils.py | 1 + 6 files changed, 43 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 9b0f8c3c32..87cdbf1d30 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -27,6 +27,16 @@ class Config(object): def __init__(self, args): pass + @staticmethod + def parse_size(string): + sizes = {"K": 1024, "M": 1024 * 1024} + size = 1 + suffix = string[-1] + if suffix in sizes: + string = string[:-1] + size = sizes[suffix] + return int(string) * size + @staticmethod def abspath(file_path): return os.path.abspath(file_path) if file_path else file_path diff --git a/synapse/config/database.py b/synapse/config/database.py index daa161c952..87efe54645 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -24,6 +24,7 @@ class DatabaseConfig(Config): self.database_path = ":memory:" else: self.database_path = self.abspath(args.database_path) + self.event_cache_size = self.parse_size(args.event_cache_size) @classmethod def add_arguments(cls, parser): @@ -33,6 +34,10 @@ class DatabaseConfig(Config): "-d", "--database-path", default="homeserver.db", help="The database name." ) + db_group.add_argument( + "--event-cache-size", default="100K", + help="Number of events to cache in memory." + ) @classmethod def generate_config(cls, args, config_dir_path): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a63c59a8a2..1170d8b6ec 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -164,6 +164,9 @@ class DataStore(RoomMemberStore, RoomStore, stream_ordering=None, is_new_state=True, current_state=None): + # Remove the any existing cache entries for the event_id + self._get_event_cache.pop(event.event_id) + # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table if current_state: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 3e1ab0a159..f13b8f4fad 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -19,6 +19,7 @@ from synapse.events import FrozenEvent from synapse.events.utils import prune_event from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext, LoggingContext +from synapse.util.lrucache import LruCache from twisted.internet import defer @@ -128,6 +129,8 @@ class SQLBaseStore(object): self._txn_perf_counters = PerformanceCounters() self._get_event_counters = PerformanceCounters() + self._get_event_cache = LruCache(hs.config.event_cache_size) + def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -579,6 +582,20 @@ class SQLBaseStore(object): def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): + + start_time = time.time() * 1000 + update_counter = self._get_event_counters.update + + try: + cache = self._get_event_cache.setdefault(event_id, {}) + # Separate cache entries for each way to invoke _get_event_txn + return cache[(check_redacted, get_prev_content, allow_rejected)] + except KeyError: + pass + finally: + start_time = update_counter("event_cache", start_time) + + sql = ( "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " "FROM event_json as e " @@ -588,7 +605,6 @@ class SQLBaseStore(object): "LIMIT 1 " ) - start_time = time.time() * 1000 txn.execute(sql, (event_id,)) @@ -599,14 +615,16 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res - self._get_event_counters.update("select_event", start_time) + start_time = update_counter("select_event", start_time) if allow_rejected or not rejected_reason: - return self._get_event_from_row_txn( + result = self._get_event_from_row_txn( txn, internal_metadata, js, redacted, check_redacted=check_redacted, get_prev_content=get_prev_content, ) + cache[(check_redacted, get_prev_content, allow_rejected)] = result + return result else: return None diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index a0a24ce096..55fbffa7a2 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -38,8 +38,9 @@ class SQLBaseStoreTestCase(unittest.TestCase): return defer.succeed(func(self.mock_txn, *args, **kwargs)) self.db_pool.runInteraction = runInteraction - hs = HomeServer("test", - db_pool=self.db_pool) + config = Mock() + config.event_cache_size = 1 + hs = HomeServer("test", db_pool=self.db_pool, config=config) self.datastore = SQLBaseStore(hs) diff --git a/tests/utils.py b/tests/utils.py index 25c33492a5..39895c739f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -41,6 +41,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): if config is None: config = Mock() config.signing_key = [MockKey()] + config.event_cache_size = 1 if datastore is None: db_pool = SQLiteMemoryDbPool() -- cgit 1.4.1 From aff892ce79b9cd5060d268eefb5876fc7caf30f0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 15:02:35 +0000 Subject: Fix formatting --- synapse/storage/_base.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f13b8f4fad..29fc334f45 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -595,7 +595,6 @@ class SQLBaseStore(object): finally: start_time = update_counter("event_cache", start_time) - sql = ( "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " "FROM event_json as e " @@ -605,7 +604,6 @@ class SQLBaseStore(object): "LIMIT 1 " ) - txn.execute(sql, (event_id,)) res = txn.fetchone() -- cgit 1.4.1 From ddc25cf4e2df10e2b175e70a2371a7e24081572f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 15:23:28 +0000 Subject: Invalidate the cache for an event if it is redacted --- synapse/storage/__init__.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1170d8b6ec..a33e2298f6 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -423,6 +423,8 @@ class DataStore(RoomMemberStore, RoomStore, ) def _store_redaction(self, txn, event): + # invalidate the cache for the redacted event + self._get_event_cache.pop(event.redacts) txn.execute( "INSERT OR IGNORE INTO redactions " "(event_id, redacts) VALUES (?,?)", -- cgit 1.4.1 From ddb816cf60ca1b0c0f9bfab5df233a010ac309a3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 15:44:28 +0000 Subject: Don't unfreeze when using FreezeEvent.get_dict, as we are using a JSONEncoder that understands FrozenDict --- synapse/events/__init__.py | 4 ---- synapse/federation/persistence.py | 7 ++++--- synapse/handlers/federation.py | 15 ++++++++++++++- synapse/handlers/room.py | 4 +++- synapse/storage/__init__.py | 11 +++++++++-- synapse/util/frozenutils.py | 8 ++++++-- 6 files changed, 36 insertions(+), 13 deletions(-) (limited to 'synapse') diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 8f0c6e959f..f4ec8cd18c 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -140,10 +140,6 @@ class FrozenEvent(EventBase): return e - def get_dict(self): - # We need to unfreeze what we return - return unfreeze(super(FrozenEvent, self).get_dict()) - def __str__(self): return self.__repr__() diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 8a1afc0ca5..76a9dcd777 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -23,7 +23,8 @@ from twisted.internet import defer from synapse.util.logutils import log_function -import simplejson as json +from syutil.jsonutil import encode_canonical_json + import logging @@ -70,7 +71,7 @@ class TransactionActions(object): transaction.transaction_id, transaction.origin, code, - json.dumps(response) + encode_canonical_json(response) ) @defer.inlineCallbacks @@ -100,5 +101,5 @@ class TransactionActions(object): transaction.transaction_id, transaction.destination, response_code, - json.dumps(response_dict) + encode_canonical_json(response_dict) ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0f9c82fd06..e36f0945ef 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -23,6 +23,7 @@ from synapse.api.errors import ( from synapse.api.constants import EventTypes, Membership, RejectedReason from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor +from synapse.util.frozenutils import unfreeze from synapse.crypto.event_signing import ( compute_event_signature, add_hashes_and_signatures, ) @@ -311,9 +312,12 @@ class FederationHandler(BaseHandler): self.room_queues[room_id] = [] builder = self.event_builder_factory.new( - event.get_pdu_json() + unfreeze(event.get_pdu_json()) ) + logger.info("Builder: %s", builder.get_pdu_json()) + logger.info("Content: %s", content) + handled_events = set() try: @@ -324,14 +328,21 @@ class FederationHandler(BaseHandler): if not hasattr(event, "signatures"): builder.signatures = {} + logger.info("Content befhahs: %s", builder.content) + add_hashes_and_signatures( builder, self.hs.hostname, self.hs.config.signing_key[0], ) + logger.info("Content aftet hah: %s", builder.content) + logger.info("Content pdu json: %s", builder.get_pdu_json()) + new_event = builder.build() + logger.info("Content after build: %s", new_event.content) + # Try the host we successfully got a response to /make_join/ # request first. try: @@ -340,6 +351,7 @@ class FederationHandler(BaseHandler): except ValueError: pass + logger.info(new_event.content) ret = yield self.replication_layer.send_join( target_hosts, new_event @@ -485,6 +497,7 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False + logger.info(event.content) context = yield self._handle_new_event(origin, event) logger.debug( diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0369b907a5..c685991a9f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -403,7 +403,9 @@ class RoomMemberHandler(BaseHandler): "membership": Membership.JOIN, "content": content, }) + logger.info("Before build: %s", builder.get_pdu_json()) event, context = yield self._create_new_client_event(builder) + logger.info("After build: %s", event.get_dict()) yield self._do_join(event, context, room_hosts=hosts, do_auth=True) @@ -462,7 +464,7 @@ class RoomMemberHandler(BaseHandler): room_hosts, room_id, event.user_id, - event.get_dict()["content"], # FIXME To get a non-frozen dict + event.content, # FIXME To get a non-frozen dict context ) else: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 924ea89035..94603ef826 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -298,12 +298,16 @@ class DataStore(RoomMemberStore, RoomStore, or_replace=True, ) + content = encode_canonical_json( + event.content + ).decode("UTF-8") + vals = { "topological_ordering": event.depth, "event_id": event.event_id, "type": event.type, "room_id": event.room_id, - "content": json.dumps(event.get_dict()["content"]), + "content": content, "processed": True, "outlier": outlier, "depth": event.depth, @@ -323,7 +327,10 @@ class DataStore(RoomMemberStore, RoomStore, "prev_events", ] } - vals["unrecognized_keys"] = json.dumps(unrec) + + vals["unrecognized_keys"] = encode_canonical_json( + unrec + ).decode("UTF-8") try: self._simple_insert_txn( diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index a13a2015e4..9e10d37aec 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -21,6 +21,9 @@ def freeze(o): if t is dict: return frozendict({k: freeze(v) for k, v in o.items()}) + if t is frozendict: + return o + if t is str or t is unicode: return o @@ -33,10 +36,11 @@ def freeze(o): def unfreeze(o): - if isinstance(o, frozendict) or isinstance(o, dict): + t = type(o) + if t is dict or t is frozendict: return dict({k: unfreeze(v) for k, v in o.items()}) - if isinstance(o, basestring): + if t is str or t is unicode: return o try: -- cgit 1.4.1 From 7be0f6594e2a6dd7c3dc745eb856025276ec7d1f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 11 Feb 2015 15:53:56 +0000 Subject: First step of making user_rooms_intersect() faster - implement in intersection logic in Python code terms of a DB query that is cacheable per user --- synapse/storage/roommember.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index c69dd995ce..d490a374e5 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -240,28 +240,30 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results + @defer.inlineCallbacks def user_rooms_intersect(self, user_id_list): """ Checks whether all the users whose IDs are given in a list share a room. + + This is a "hot path" function that's called a lot, e.g. by presence for + generating the event stream. """ - def interaction(txn): - user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) - sql = ( - "SELECT m.room_id FROM room_memberships as m " - "INNER JOIN current_state_events as c " - "ON m.event_id = c.event_id " - "WHERE m.membership = 'join' " - "AND (%(clause)s) " - # TODO(paul): We've got duplicate rows in the database somewhere - # so we have to DISTINCT m.user_id here - "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" - ) % {"clause": user_list_clause} - - args = list(user_id_list) - args.append(len(user_id_list)) + if len(user_id_list) < 2: + defer.returnValue(True) - txn.execute(sql, args) + deferreds = [ + self.get_rooms_for_user_where_membership_is( + u, membership_list=[Membership.JOIN], + ) + for u in user_id_list + ] + + results = yield defer.DeferredList(deferreds) + + # A list of sets of strings giving room IDs for each user + room_id_lists = [set([r.room_id for r in result[1]]) for result in results] - return len(txn.fetchall()) > 0 + # There isn't a setintersection(*list_of_sets) + ret = len(room_id_lists.pop(0).intersection(*room_id_lists)) > 0 - return self.runInteraction("user_rooms_intersect", interaction) + defer.returnValue(ret) -- cgit 1.4.1 From 45b56609ae84e7ffc3713335e7d9abc315ad1725 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 11 Feb 2015 16:04:08 +0000 Subject: Cache the result of a get_rooms_for_user query, to make user_rooms_intersect() much lighter in the read-common case --- synapse/storage/roommember.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index d490a374e5..e05465bc13 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -35,6 +35,11 @@ RoomsForUser = namedtuple( class RoomMemberStore(SQLBaseStore): + def __init__(self, *args, **kw): + super(RoomMemberStore, self).__init__(*args, **kw) + + self._user_rooms_cache = {} + def _store_room_member_txn(self, txn, event): """Store a room member in the database. """ @@ -98,6 +103,8 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, (event.room_id, domain)) + self.invalidate_rooms_for_user(target_user_id) + @defer.inlineCallbacks def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. @@ -240,23 +247,43 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results + # TODO(paul): Create a nice @cached decorator to do this + # @cached + # def get_foo(...) + # ... + # invalidate_foo = get_foo.invalidator + + @defer.inlineCallbacks + def get_rooms_for_user(self, user_id): + # TODO(paul): put some performance counters in here so we can easily + # track what impact this cache is having + if user_id in self._user_rooms_cache: + defer.returnValue(self._user_rooms_cache[user_id]) + + rooms = yield self.get_rooms_for_user_where_membership_is( + user_id, membership_list=[Membership.JOIN], + ) + + self._user_rooms_cache[user_id] = rooms + defer.returnValue(rooms) + + def invalidate_rooms_for_user(self, user_id): + if user_id in self._user_rooms_cache: + del self._user_rooms_cache[user_id] + @defer.inlineCallbacks def user_rooms_intersect(self, user_id_list): """ Checks whether all the users whose IDs are given in a list share a room. This is a "hot path" function that's called a lot, e.g. by presence for - generating the event stream. + generating the event stream. As such, it is implemented locally by + wrapping logic around heavily-cached database queries. """ if len(user_id_list) < 2: defer.returnValue(True) - deferreds = [ - self.get_rooms_for_user_where_membership_is( - u, membership_list=[Membership.JOIN], - ) - for u in user_id_list - ] + deferreds = [self.get_rooms_for_user(u) for u in user_id_list] results = yield defer.DeferredList(deferreds) -- cgit 1.4.1 From 7f47ba7383302fdbdaa3a10abef00d3710c77fce Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 11 Feb 2015 16:18:21 +0000 Subject: Added another TODO note --- synapse/storage/roommember.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'synapse') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index e05465bc13..779f9ce544 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -264,6 +264,9 @@ class RoomMemberStore(SQLBaseStore): user_id, membership_list=[Membership.JOIN], ) + # TODO(paul): Consider applying a maximum size; just evict things at + # random, or consider LRU? + self._user_rooms_cache[user_id] = rooms defer.returnValue(rooms) -- cgit 1.4.1 From f51832442674a1f72c41ffce2279880109fc7ff0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 16:41:16 +0000 Subject: Minor tweaks based on PR feedback. --- synapse/appservice/api.py | 6 +++--- synapse/http/client.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 6192813c03..c2179f8d55 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -80,11 +80,11 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.put_json( - uri, - { + uri=uri, + json_body={ "events": events }, - { + args={ "access_token": service.hs_token }) if response: # just an empty json object diff --git a/synapse/http/client.py b/synapse/http/client.py index 575510637e..7b23116556 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -95,7 +95,8 @@ class SimpleHttpClient(object): Deferred: Succeeds when we get *any* 2xx HTTP response, with the HTTP body as JSON. Raises: - On a non-2xx HTTP response. + On a non-2xx HTTP response. The response body will be used as the + error message. """ if len(args): query_bytes = urllib.urlencode(args, True) -- cgit 1.4.1 From ef276e8770e19c66d14462aa325b9cb241121bb6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 16:48:05 +0000 Subject: Fix so timing out connections to actually work. --- synapse/federation/replication.py | 2 ++ synapse/util/__init__.py | 52 ++++++++++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index e442c6c5d5..54a0c7ad8e 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -72,5 +72,7 @@ class ReplicationLayer(FederationClient, FederationServer): self._order = 0 + self.hs = hs + def __str__(self): return "" % self.server_name diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 29f1344c5b..e77eba90ad 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -18,6 +18,9 @@ from synapse.util.logcontext import LoggingContext from twisted.internet import defer, reactor, task import time +import logging + +logger = logging.getLogger(__name__) class Clock(object): @@ -55,20 +58,51 @@ class Clock(object): timer.cancel() def time_bound_deferred(self, given_deferred, time_out): + if given_deferred.called: + return given_deferred + ret_deferred = defer.Deferred() - def timed_out(): - if not given_deferred.called: - given_deferred.cancel() + def timed_out_fn(): + try: ret_deferred.errback(RuntimeError("Timed out")) + except: + pass + + try: + given_deferred.cancel() + except: + pass + + timer = None + + def cancel(res): + try: + self.cancel_call_later(timer) + except: + pass + return res + + ret_deferred.addBoth(cancel) + + def sucess(res): + try: + ret_deferred.callback(res) + except: + pass + + return res + + def err(res): + try: + ret_deferred.errback(res) + except: + pass - timer = self.call_later(time_out, timed_out) + return res - def succeed(result): - self.cancel_call_later(timer) - ret_deferred.callback(result) + given_deferred.addCallbacks(callback=sucess, errback=err) - given_deferred.addCallback(succeed) - given_deferred.addErrback(ret_deferred.errback) + timer = self.call_later(time_out, timed_out_fn) return ret_deferred -- cgit 1.4.1 From ed877d658500b736e7f7192fbeb64aac444ef4d8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 16:50:46 +0000 Subject: Remove debug logging --- synapse/handlers/federation.py | 10 ---------- synapse/handlers/room.py | 2 -- 2 files changed, 12 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e36f0945ef..0c35317bb8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -315,9 +315,6 @@ class FederationHandler(BaseHandler): unfreeze(event.get_pdu_json()) ) - logger.info("Builder: %s", builder.get_pdu_json()) - logger.info("Content: %s", content) - handled_events = set() try: @@ -328,21 +325,14 @@ class FederationHandler(BaseHandler): if not hasattr(event, "signatures"): builder.signatures = {} - logger.info("Content befhahs: %s", builder.content) - add_hashes_and_signatures( builder, self.hs.hostname, self.hs.config.signing_key[0], ) - logger.info("Content aftet hah: %s", builder.content) - logger.info("Content pdu json: %s", builder.get_pdu_json()) - new_event = builder.build() - logger.info("Content after build: %s", new_event.content) - # Try the host we successfully got a response to /make_join/ # request first. try: diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c685991a9f..914742d913 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -403,9 +403,7 @@ class RoomMemberHandler(BaseHandler): "membership": Membership.JOIN, "content": content, }) - logger.info("Before build: %s", builder.get_pdu_json()) event, context = yield self._create_new_client_event(builder) - logger.info("After build: %s", event.get_dict()) yield self._do_join(event, context, room_hosts=hosts, do_auth=True) -- cgit 1.4.1 From 76935078d1983aec9966f2eb245e4186dcee0386 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 16:51:22 +0000 Subject: Remove more debug logging --- synapse/handlers/federation.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0c35317bb8..77c81fe2da 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -341,7 +341,6 @@ class FederationHandler(BaseHandler): except ValueError: pass - logger.info(new_event.content) ret = yield self.replication_layer.send_join( target_hosts, new_event @@ -487,7 +486,6 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False - logger.info(event.content) context = yield self._handle_new_event(origin, event) logger.debug( -- cgit 1.4.1 From 7eef84a95b53318e97ab74dd25fee584714a8e6a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 16:52:22 +0000 Subject: pyflakes --- synapse/events/__init__.py | 2 +- synapse/storage/__init__.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index f4ec8cd18c..64e08223b0 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.util.frozenutils import freeze, unfreeze +from synapse.util.frozenutils import freeze class _EventInternalMetadata(object): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 94603ef826..a0284d54b9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -44,7 +44,6 @@ from syutil.jsonutil import encode_canonical_json from synapse.crypto.event_signing import compute_event_reference_hash -import simplejson as json import logging import os -- cgit 1.4.1 From 9c24cff6ef789f277e3c6663d356abb3c915aaa8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 17:00:20 +0000 Subject: Allow newer versions of syutil --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index e2a9d1f6a7..c7a685e71e 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -4,7 +4,7 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil==0.0.2": ["syutil"], + "syutil>=0.0.2": ["syutil"], "matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], -- cgit 1.4.1 From a793a0b810f0f5d8fa4018b52d98471cc32fdf80 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 17:02:52 +0000 Subject: Bump syutil version --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index c7a685e71e..8164effeee 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -4,7 +4,7 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil>=0.0.2": ["syutil"], + "syutil>=0.0.3": ["syutil"], "matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], -- cgit 1.4.1 From af89456c3c6a0dbc871c2c535840c5f95c33df32 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 17:15:16 +0000 Subject: Update dependency links --- synapse/python_dependencies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 8164effeee..e099a42b5f 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -26,8 +26,8 @@ def github_link(project, version, egg): DEPENDENCY_LINKS = [ github_link( project="matrix-org/syutil", - version="v0.0.2", - egg="syutil-0.0.2", + version="v0.0.3", + egg="syutil-0.0.3", ), github_link( project="matrix-org/matrix-angular-sdk", -- cgit 1.4.1 From e9e54449f5ff830c36c7ec6935e723ae68dca942 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 17:32:42 +0000 Subject: Use encode_canonical_json for pushes --- synapse/push/pusherpool.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 7483d257bf..90babd7224 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -19,6 +19,8 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException +from syutil.jsonutil import encode_canonical_json + import logging import simplejson as json @@ -96,7 +98,7 @@ class PusherPool: pushkey=pushkey, pushkey_ts=self.hs.get_clock().time_msec(), lang=lang, - data=json.dumps(data) + data=encode_canonical_json(data).decode("UTF-8"), ) self._refresh_pusher((app_id, pushkey)) -- cgit 1.4.1 From 2bf0e85f3d43d41f450df4c475c9deb542999d82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 17:34:23 +0000 Subject: Use encode_canonical_json for http client --- synapse/http/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/http/client.py b/synapse/http/client.py index d500e19c81..43b2329780 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -15,6 +15,8 @@ from synapse.http.agent_name import AGENT_NAME +from syutil.jsonutil import encode_canonical_json + from twisted.internet import defer, reactor from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError @@ -64,7 +66,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def post_json_get_json(self, uri, post_json): - json_str = json.dumps(post_json) + json_str = encode_canonical_json(post_json) logger.info("HTTP POST %s -> %s", json_str, uri) -- cgit 1.4.1 From 8bbdf328498ef9a7e6c46e565195783b208bfeb2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 18:56:13 +0000 Subject: Convert get_rooms to use runInteraction so the transacion has a more helpful description --- synapse/storage/room.py | 67 +++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 30 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 6542f8e4f8..750b17a45f 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -82,38 +82,45 @@ class RoomStore(SQLBaseStore): "topic" key if one is set, and a "name" key if one is set """ - topic_subquery = ( - "SELECT topics.event_id as event_id, " - "topics.room_id as room_id, topic " - "FROM topics " - "INNER JOIN current_state_events as c " - "ON c.event_id = topics.event_id " - ) + def f(txn): + topic_subquery = ( + "SELECT topics.event_id as event_id, " + "topics.room_id as room_id, topic " + "FROM topics " + "INNER JOIN current_state_events as c " + "ON c.event_id = topics.event_id " + ) - name_subquery = ( - "SELECT room_names.event_id as event_id, " - "room_names.room_id as room_id, name " - "FROM room_names " - "INNER JOIN current_state_events as c " - "ON c.event_id = room_names.event_id " - ) + name_subquery = ( + "SELECT room_names.event_id as event_id, " + "room_names.room_id as room_id, name " + "FROM room_names " + "INNER JOIN current_state_events as c " + "ON c.event_id = room_names.event_id " + ) - # We use non printing ascii character US () as a seperator - sql = ( - "SELECT r.room_id, n.name, t.topic, " - "group_concat(a.room_alias, '') " - "FROM rooms AS r " - "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " - "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " - "INNER JOIN room_aliases AS a ON a.room_id = r.room_id " - "WHERE r.is_public = ? " - "GROUP BY r.room_id " - ) % { - "topic": topic_subquery, - "name": name_subquery, - } - - rows = yield self._execute(None, sql, is_public) + # We use non printing ascii character US () as a seperator + sql = ( + "SELECT r.room_id, n.name, t.topic, " + "group_concat(a.room_alias, '') " + "FROM rooms AS r " + "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " + "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " + "INNER JOIN room_aliases AS a ON a.room_id = r.room_id " + "WHERE r.is_public = ? " + "GROUP BY r.room_id " + ) % { + "topic": topic_subquery, + "name": name_subquery, + } + + c = txn.execute(sql, (is_public,)) + + return c.fetchall() + + rows = yield self.runInteraction( + "get_rooms", f + ) ret = [ { -- cgit 1.4.1 From ff2a2ae56ecd68c180e1d2548dffbe068d61f70f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:02:06 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 8fe8df4edb..915af3fe09 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.6.1f" +__version__ = "0.7.0" -- cgit 1.4.1 From 83d41f25d8c57a2fa3a5feed4b952690abbd57a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:05:47 +0000 Subject: Set database schema version in delta --- synapse/storage/schema/delta/v12.sql | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse') diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql index 302d958dbf..b87ef1fe79 100644 --- a/synapse/storage/schema/delta/v12.sql +++ b/synapse/storage/schema/delta/v12.sql @@ -63,3 +63,5 @@ CREATE TABLE IF NOT EXISTS user_filters( CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id ); + +PRAGMA user_version = 12; -- cgit 1.4.1 From a566ed2f0ea09b4a1874f2bf4b1014e05a308e8b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:49:07 +0000 Subject: Bump webclient version --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index e099a42b5f..ec78fc3627 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil>=0.0.3": ["syutil"], - "matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"], + "matrix_angular_sdk>=0.6.2": ["syweb>=0.6.2"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -31,8 +31,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/matrix-angular-sdk", - version="v0.6.1", - egg="matrix_angular_sdk-0.6.1", + version="v0.6.2", + egg="matrix_angular_sdk-0.6.2", ), github_link( project="pyca/pynacl", -- cgit 1.4.1 From 717687e1fc61a0fa441ff0dee2d49fda518c06bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 14:39:31 +0000 Subject: Get an auth query one at a time --- synapse/storage/event_federation.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 0cbcdd1b55..3fbc090224 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -55,17 +55,16 @@ class EventFederationStore(SQLBaseStore): results = set() base_sql = ( - "SELECT auth_id FROM event_auth WHERE %s" + "SELECT auth_id FROM event_auth WHERE event_id = ?" ) front = set(event_ids) while front: - sql = base_sql % ( - " OR ".join(["event_id=?"] * len(front)), - ) - - txn.execute(sql, list(front)) - front = [r[0] for r in txn.fetchall()] + new_front = set() + for f in front: + txn.execute(base_sql, (f,)) + new_front.update([r[0] for r in txn.fetchall()]) + front = new_front results.update(front) return list(results) -- cgit 1.4.1 From 48fbe79f7183f0aa7e4473a573658fe2a2f9bcd3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 14:52:02 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 915af3fe09..10147d88dc 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0" +__version__ = "0.7.0a" -- cgit 1.4.1 From 963256638d5b3c3edee14bfbd7c00944b45d04c0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 18:17:11 +0000 Subject: Correctly handle all the places that can throw exceptions --- scripts/federation_client.py | 7 +++++-- synapse/federation/federation_base.py | 24 ++++++++++++++---------- synapse/federation/federation_server.py | 9 ++++++--- synapse/handlers/federation.py | 12 ++++++++++++ 4 files changed, 37 insertions(+), 15 deletions(-) (limited to 'synapse') diff --git a/scripts/federation_client.py b/scripts/federation_client.py index 3139c61761..ea62dceb36 100644 --- a/scripts/federation_client.py +++ b/scripts/federation_client.py @@ -97,8 +97,11 @@ def lookup(destination, path): if ":" in destination: return "https://%s%s" % (destination, path) else: - srv = srvlookup.lookup("matrix", "tcp", destination)[0] - return "https://%s:%d%s" % (srv.host, srv.port, path) + try: + srv = srvlookup.lookup("matrix", "tcp", destination)[0] + return "https://%s:%d%s" % (srv.host, srv.port, path) + except: + return "https://%s:%d%s" % (destination, 8448, path) def get_json(origin_name, origin_key, destination, path): request_json = { diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index a990aec4fd..30f3f5c8a4 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -61,7 +61,8 @@ class FederationBase(object): # Check local db. new_pdu = yield self.store.get_event( pdu.event_id, - allow_rejected=True + allow_rejected=True, + allow_none=True, ) if new_pdu: signed_pdus.append(new_pdu) @@ -69,15 +70,18 @@ class FederationBase(object): # Check pdu.origin if pdu.origin != origin: - new_pdu = yield self.get_pdu( - destinations=[pdu.origin], - event_id=pdu.event_id, - outlier=outlier, - ) - - if new_pdu: - signed_pdus.append(new_pdu) - continue + try: + new_pdu = yield self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + ) + + if new_pdu: + signed_pdus.append(new_pdu) + continue + except: + pass logger.warn("Failed to find copy of %s with valid signature") diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index b23f72c7fa..9f5c98694c 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -411,9 +411,12 @@ class FederationServer(FederationBase): "_handle_new_pdu getting state for %s", pdu.room_id ) - state, auth_chain = yield self.get_state_for_room( - origin, pdu.room_id, pdu.event_id, - ) + try: + state, auth_chain = yield self.get_state_for_room( + origin, pdu.room_id, pdu.event_id, + ) + except: + logger.warn("Failed to get state for event: %s", pdu.event_id) ret = yield self.handler.on_receive_pdu( origin, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 77c81fe2da..5a7593cc3b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -788,6 +788,18 @@ class FederationHandler(BaseHandler): defer.returnValue(ret) + @defer.inlineCallbacks + def trigger_query_auth(self, destination, event_id): + local_auth_chain = yield self.store.get_auth_chain([event_id]) + + result = yield self.replication_layer.query_auth( + destination, + event.room_id, + event.event_id, + local_auth_chain, + ) + + @defer.inlineCallbacks @log_function def do_auth(self, origin, event, context, auth_events): -- cgit 1.4.1 From 58d848adc0ee9ab8e44a780a19be1e46e0d5cec3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 18:35:36 +0000 Subject: Parrellize fetching of events --- synapse/federation/federation_base.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 30f3f5c8a4..966f4d2c5c 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -50,8 +50,11 @@ class FederationBase(object): Returns: Deferred : A list of PDUs that have valid signatures and hashes. """ + signed_pdus = [] - for pdu in pdus: + + @defer.inlineCallbacks + def do(pdu): try: new_pdu = yield self._check_sigs_and_hash(pdu) signed_pdus.append(new_pdu) @@ -66,7 +69,7 @@ class FederationBase(object): ) if new_pdu: signed_pdus.append(new_pdu) - continue + return # Check pdu.origin if pdu.origin != origin: @@ -79,12 +82,17 @@ class FederationBase(object): if new_pdu: signed_pdus.append(new_pdu) - continue + return except: pass logger.warn("Failed to find copy of %s with valid signature") + yield defer.gatherResults( + [do(pdu) for pdu in pdus], + consumeErrors=True + ) + defer.returnValue(signed_pdus) @defer.inlineCallbacks -- cgit 1.4.1 From 860b1b4841618ec401c3652dcf19ac8b5d167f50 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:11:52 +0000 Subject: Only attempt to fetch presence state of JOINed members in room initialSync (SYN-202) --- synapse/handlers/message.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3355adefcf..c6d6188d9d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -372,6 +372,7 @@ class MessageHandler(BaseHandler): room_members = [ m for m in current_state.values() if m.type == EventTypes.Member + and m.content["membership"] == Membership.JOIN ] presence_handler = self.hs.get_handlers().presence_handler -- cgit 1.4.1 From 29805213d1c20c89c20a4b3075c30597b4402338 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:12:57 +0000 Subject: Can now remove the FIXME too --- synapse/handlers/message.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c6d6188d9d..5f4cc1a84b 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -386,16 +386,9 @@ class MessageHandler(BaseHandler): ) presence.append(member_presence) except SynapseError as e: - if e.code == 404: - # FIXME: We are doing this as a warn since this gets hit a - # lot and spams the logs. Why is this happening? - logger.warn( - "Failed to get member presence of %r", m.user_id - ) - else: - logger.exception( - "Failed to get member presence of %r", m.user_id - ) + logger.exception( + "Failed to get member presence of %r", m.user_id + ) time_now = self.clock.time_msec() -- cgit 1.4.1 From 5f6e6530d0f43c637e527cfe1797801879dbc389 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:15:23 +0000 Subject: Appease pyflakes --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5f4cc1a84b..e028cfc2e9 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -385,7 +385,7 @@ class MessageHandler(BaseHandler): as_event=True, ) presence.append(member_presence) - except SynapseError as e: + except SynapseError: logger.exception( "Failed to get member presence of %r", m.user_id ) -- cgit 1.4.1 From 93d07c87dc9b837492fa0f691a77e716a2fe512b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:19:37 +0000 Subject: Reindent code to be less human-readable to keep pep8 from complaining --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index e028cfc2e9..7b9685be7f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -372,7 +372,7 @@ class MessageHandler(BaseHandler): room_members = [ m for m in current_state.values() if m.type == EventTypes.Member - and m.content["membership"] == Membership.JOIN + and m.content["membership"] == Membership.JOIN ] presence_handler = self.hs.get_handlers().presence_handler -- cgit 1.4.1 From 38df10b99eaa7f8165c9e541142e96bf023ad744 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 19:29:32 +0000 Subject: Remove unused function --- synapse/handlers/federation.py | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5a7593cc3b..77c81fe2da 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -788,18 +788,6 @@ class FederationHandler(BaseHandler): defer.returnValue(ret) - @defer.inlineCallbacks - def trigger_query_auth(self, destination, event_id): - local_auth_chain = yield self.store.get_auth_chain([event_id]) - - result = yield self.replication_layer.query_auth( - destination, - event.room_id, - event.event_id, - local_auth_chain, - ) - - @defer.inlineCallbacks @log_function def do_auth(self, origin, event, context, auth_events): -- cgit 1.4.1 From 789251afa7d97a08318f92ffe96f5f7ce16d6157 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 19:29:43 +0000 Subject: Fix logging --- synapse/federation/federation_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 966f4d2c5c..21a763214b 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -86,7 +86,10 @@ class FederationBase(object): except: pass - logger.warn("Failed to find copy of %s with valid signature") + logger.warn( + "Failed to find copy of %s with valid signature", + pdu.event_id, + ) yield defer.gatherResults( [do(pdu) for pdu in pdus], -- cgit 1.4.1 From f22646efcc3d3c5c7fc0d7797de2675e1a794dd0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:11:52 +0000 Subject: Only attempt to fetch presence state of JOINed members in room initialSync (SYN-202) --- synapse/handlers/message.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3355adefcf..c6d6188d9d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -372,6 +372,7 @@ class MessageHandler(BaseHandler): room_members = [ m for m in current_state.values() if m.type == EventTypes.Member + and m.content["membership"] == Membership.JOIN ] presence_handler = self.hs.get_handlers().presence_handler -- cgit 1.4.1 From 55e0916ffc0a4e2c7bff533c561e99889650c61d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:19:37 +0000 Subject: Reindent code to be less human-readable to keep pep8 from complaining --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c6d6188d9d..b00380387d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -372,7 +372,7 @@ class MessageHandler(BaseHandler): room_members = [ m for m in current_state.values() if m.type == EventTypes.Member - and m.content["membership"] == Membership.JOIN + and m.content["membership"] == Membership.JOIN ] presence_handler = self.hs.get_handlers().presence_handler -- cgit 1.4.1 From c09493d7aa79386645ae48f1232158f99dd6049a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 13:33:37 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 10147d88dc..c0aa2a757f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0a" +__version__ = "0.7.0b" -- cgit 1.4.1 From 8c652a2b5f4ed07abd682543ad6a365c6d50b8d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 14:20:05 +0000 Subject: When we see a difference in current state, actually use state conflict resolution algorithm --- synapse/handlers/federation.py | 34 +++++++++++++++++++++++++++++++++ synapse/state.py | 43 ++++++++++++++++++++++++++++-------------- 2 files changed, 63 insertions(+), 14 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 77c81fe2da..5b225ffb57 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -858,6 +858,40 @@ class FederationHandler(BaseHandler): # Do auth conflict res. logger.debug("Different auth: %s", different_auth) + different_events = yield defer.gatherResults( + [ + self.store.get_event( + d, + allow_none=True, + allow_rejected=False, + ) + for d in different_auth + if d in have_events and not have_events[d] + ], + consumeErrors=True + ) + + if different_events: + local_view = dict(auth_events) + remote_view = dict(auth_events) + remote_view.update({ + (d.type, d.state_key) for d in different_events + }) + + new_state, _ = self.state.resolve_events( + [local_view, remote_view], + event + ) + + auth_events.update(new_state) + + current_state = set(e.event_id for e in auth_events.values()) + different_auth = event_auth_events - current_state + + context.current_state.update(auth_events) + context.state_group = None + + if different_auth and not event.internal_metadata.is_outlier(): # Only do auth resolution if we have something new to say. # We can't rove an auth failure. do_resolution = False diff --git a/synapse/state.py b/synapse/state.py index 98aaa2be53..fe5f3dc84b 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -259,13 +259,37 @@ class StateHandler(object): defer.returnValue((name, state, prev_states)) + new_state, prev_states = self._resolve_events( + state_groups.values(), event_type, state_key + ) + + if self._state_cache is not None: + cache = _StateCacheEntry( + state=new_state, + state_group=None, + ts=self.clock.time_msec() + ) + + self._state_cache[frozenset(event_ids)] = cache + + defer.returnValue((None, new_state, prev_states)) + + def resolve_events(self, state_sets, event): + if event.is_state(): + return self._resolve_events( + state_sets, event.type, event.state_key + ) + else: + return self._resolve_events(state_sets) + + def _resolve_events(self, state_sets, event_type=None, state_key=""): state = {} - for group, g_state in state_groups.items(): - for s in g_state: + for st in state_sets: + for e in st: state.setdefault( - (s.type, s.state_key), + (e.type, e.state_key), {} - )[s.event_id] = s + )[e.event_id] = e unconflicted_state = { k: v.values()[0] for k, v in state.items() @@ -302,16 +326,7 @@ class StateHandler(object): new_state = unconflicted_state new_state.update(resolved_state) - if self._state_cache is not None: - cache = _StateCacheEntry( - state=new_state, - state_group=None, - ts=self.clock.time_msec() - ) - - self._state_cache[frozenset(event_ids)] = cache - - defer.returnValue((None, new_state, prev_states)) + return new_state, prev_states @log_function def _resolve_state_events(self, conflicted_state, auth_events): -- cgit 1.4.1 From e441c10a7302a3b0ddfa3caa7823fa6f62372f82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 14:23:39 +0000 Subject: pyflakes --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5b225ffb57..d667d358ab 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -878,7 +878,7 @@ class FederationHandler(BaseHandler): (d.type, d.state_key) for d in different_events }) - new_state, _ = self.state.resolve_events( + new_state, prev_state = self.state.resolve_events( [local_view, remote_view], event ) -- cgit 1.4.1 From 0546126cc5fb1a138c96481061084e9e5fc86e34 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 14:36:40 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index c0aa2a757f..1060fcc866 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0b" +__version__ = "0.7.0c" -- cgit 1.4.1 From 183b3d4e47dd2fc0e0ca88714d0e0bd415f81736 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 Feb 2015 14:29:49 +0000 Subject: Prepare the database whenever a connection is opened from the db_pool so that in-memory databases will work --- synapse/app/homeserver.py | 14 ++++---------- synapse/storage/__init__.py | 3 +++ 2 files changed, 7 insertions(+), 10 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f5681fac20..6f39819d3a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -90,7 +90,9 @@ class SynapseHomeServer(HomeServer): "sqlite3", self.get_db_name(), check_same_thread=False, cp_min=1, - cp_max=1 + cp_max=1, + cp_openfun=prepare_database, # Prepare the database for each conn + # so that :memory: sqlite works ) def create_resource_tree(self, web_client, redirect_root_to_web_client): @@ -252,14 +254,6 @@ def setup(): logger.info("Database prepared in %s.", db_name) - db_pool = hs.get_db_pool() - - if db_name == ":memory:": - # Memory databases will need to be setup each time they are opened. - reactor.callWhenRunning( - db_pool.runWithConnection, prepare_database - ) - if config.manhole: f = twisted.manhole.telnet.ShellFactory() f.username = "matrix" @@ -270,10 +264,10 @@ def setup(): bind_port = config.bind_port if config.no_tls: bind_port = None + hs.start_listening(bind_port, config.unsecure_port) hs.get_pusherpool().start() - hs.get_state_handler().start_caching() hs.get_datastore().start_profiling() diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 02b1f06854..1c22e19ab0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -637,10 +637,13 @@ def prepare_database(db_conn): c.executescript(sql_script) db_conn.commit() + else: + logger.info("Database is at version %r", user_version) else: sql_script = "BEGIN TRANSACTION;\n" for sql_loc in SCHEMAS: + logger.debug("Applying schema %r", sql_loc) sql_script += read_schema(sql_loc) sql_script += "\n" sql_script += "COMMIT TRANSACTION;" -- cgit 1.4.1 From b11a6e1c3ca51e81b911fa733a4169bd6cbaa5ee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 15:37:18 +0000 Subject: Fix wrong variable name --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index d667d358ab..4cd795e202 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -878,7 +878,7 @@ class FederationHandler(BaseHandler): (d.type, d.state_key) for d in different_events }) - new_state, prev_state = self.state.resolve_events( + new_state, prev_state = self.state_handler.resolve_events( [local_view, remote_view], event ) -- cgit 1.4.1 From be26697b29a44b96cb0ed371ca4aaab2654d6802 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 15:37:35 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 1060fcc866..7ba21cd247 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0c" +__version__ = "0.7.0d" -- cgit 1.4.1 From 8d5cce62ab4bbb2333f01abb9edf7e94f6d6f7d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 13 Feb 2015 16:16:16 +0000 Subject: Update pushers by app id and pushkey, not user id and pushkey --- synapse/push/__init__.py | 12 ++++++------ synapse/storage/pusher.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 0659a1cb9b..0fb3e4f7f3 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -237,7 +237,7 @@ class Pusher(object): self.user_name, config, timeout=0) self.last_token = chunk['end'] self.store.update_pusher_last_token( - self.user_name, self.pushkey, self.last_token) + self.app_id, self.pushkey, self.last_token) logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) @@ -308,7 +308,7 @@ class Pusher(object): self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token_and_success( - self.user_name, + self.app_id, self.pushkey, self.last_token, self.clock.time_msec() @@ -316,14 +316,14 @@ class Pusher(object): if self.failing_since: self.failing_since = None self.store.update_pusher_failing_since( - self.user_name, + self.app_id, self.pushkey, self.failing_since) else: if not self.failing_since: self.failing_since = self.clock.time_msec() self.store.update_pusher_failing_since( - self.user_name, + self.app_id, self.pushkey, self.failing_since ) @@ -340,14 +340,14 @@ class Pusher(object): self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token( - self.user_name, + self.app_id, self.pushkey, self.last_token ) self.failing_since = None self.store.update_pusher_failing_since( - self.user_name, + self.app_id, self.pushkey, self.failing_since ) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index e2a662a6c7..6622b4d18a 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -126,27 +126,27 @@ class PusherStore(SQLBaseStore): ) @defer.inlineCallbacks - def update_pusher_last_token(self, user_name, pushkey, last_token): + def update_pusher_last_token(self, app_id, pushkey, last_token): yield self._simple_update_one( PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey}, {'last_token': last_token} ) @defer.inlineCallbacks - def update_pusher_last_token_and_success(self, user_name, pushkey, + def update_pusher_last_token_and_success(self, app_id, pushkey, last_token, last_success): yield self._simple_update_one( PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey}, {'last_token': last_token, 'last_success': last_success} ) @defer.inlineCallbacks - def update_pusher_failing_since(self, user_name, pushkey, failing_since): + def update_pusher_failing_since(self, app_id, pushkey, failing_since): yield self._simple_update_one( PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey}, {'failing_since': failing_since} ) -- cgit 1.4.1 From a78838c5ba5c9342bb3dc53e0c814b00d22be658 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 16:17:54 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 7ba21cd247..1a02aefe26 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0d" +__version__ = "0.7.0e" -- cgit 1.4.1 From b99a33f283d595f5f462ab039d672ca04bdffcb8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sun, 15 Feb 2015 20:20:51 +0000 Subject: resolve_events expect lists, not dicts --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4cd795e202..3aecc1ca51 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -879,7 +879,7 @@ class FederationHandler(BaseHandler): }) new_state, prev_state = self.state_handler.resolve_events( - [local_view, remote_view], + [local_view.values(), remote_view.values()], event ) -- cgit 1.4.1 From 6dd5c958412ac9c589996c8c5d92eeacdbeb734b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sun, 15 Feb 2015 20:38:52 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 1a02aefe26..792b8089f6 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0e" +__version__ = "0.7.0f" -- cgit 1.4.1 From 61385846510ceed1bff571dde1233a70e02b4748 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Feb 2015 14:08:02 +0000 Subject: Don't return anything from _handle_new_pdu, since we ignore the return value anyway --- synapse/federation/federation_server.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 9f5c98694c..7ca8d49c54 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -331,7 +331,6 @@ class FederationServer(FederationBase): ) if already_seen: logger.debug("Already seen pdu %s", pdu.event_id) - defer.returnValue({}) return # Check signature. @@ -418,7 +417,7 @@ class FederationServer(FederationBase): except: logger.warn("Failed to get state for event: %s", pdu.event_id) - ret = yield self.handler.on_receive_pdu( + yield self.handler.on_receive_pdu( origin, pdu, backfilled=False, @@ -426,8 +425,6 @@ class FederationServer(FederationBase): auth_chain=auth_chain, ) - defer.returnValue(ret) - def __str__(self): return "" % self.server_name -- cgit 1.4.1 From 91fc5eef1deb2f756d16f3d5ea20ede8d967f6df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Feb 2015 14:27:40 +0000 Subject: Mark old events as outliers. This is to fix the issue where if a remote server sends an event that references a really "old" event, then the local server will pull that in and send to all clients. We decide if an event is old if its depth is less than the minimum depth of the room. --- synapse/federation/federation_server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 7ca8d49c54..4391a60c02 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -366,7 +366,13 @@ class FederationServer(FederationBase): pdu.room_id, min_depth ) - if min_depth and pdu.depth > min_depth and max_recursion > 0: + if min_depth and pdu.depth < min_depth: + # This is so that we don't notify the user about this + # message, to work around the fact that some events will + # reference really really old events we really don't want to + # send to the clients. + pdu.internal_metadata.outlier = True + elif min_depth and pdu.depth > min_depth and max_recursion > 0: for event_id, hashes in pdu.prev_events: if event_id not in have_seen: logger.debug( -- cgit 1.4.1 From 2674aeb96a1b75583dc3ea514cbce580e8ae35c8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Feb 2015 16:16:47 +0000 Subject: Factor out ExpiringCache from StateHandler --- synapse/state.py | 46 ++++------------- synapse/util/expiringcache.py | 115 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 37 deletions(-) create mode 100644 synapse/util/expiringcache.py (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index fe5f3dc84b..80cced351d 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor +from synapse.util.expiringcache import ExpiringCache from synapse.api.constants import EventTypes from synapse.api.errors import AuthError from synapse.events.snapshot import EventContext @@ -51,7 +52,6 @@ class _StateCacheEntry(object): def __init__(self, state, state_group, ts): self.state = state self.state_group = state_group - self.ts = ts class StateHandler(object): @@ -69,12 +69,15 @@ class StateHandler(object): def start_caching(self): logger.debug("start_caching") - self._state_cache = {} - - def f(): - self._prune_cache() + self._state_cache = ExpiringCache( + cache_name="state_cache", + clock=self.clock, + max_len=SIZE_OF_CACHE, + expiry_ms=EVICTION_TIMEOUT_SECONDS*1000, + reset_expiry_on_get=True, + ) - self.clock.looping_call(f, 5*1000) + self._state_cache.start() @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): @@ -409,34 +412,3 @@ class StateHandler(object): return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() return sorted(events, key=key_func) - - def _prune_cache(self): - logger.debug( - "_prune_cache. before len: %d", - len(self._state_cache.keys()) - ) - - now = self.clock.time_msec() - - if len(self._state_cache.keys()) > SIZE_OF_CACHE: - sorted_entries = sorted( - self._state_cache.items(), - key=lambda k, v: v.ts, - ) - - for k, _ in sorted_entries[SIZE_OF_CACHE:]: - self._state_cache.pop(k) - - keys_to_delete = set() - - for key, cache_entry in self._state_cache.items(): - if now - cache_entry.ts > EVICTION_TIMEOUT_SECONDS*1000: - keys_to_delete.add(key) - - for k in keys_to_delete: - self._state_cache.pop(k) - - logger.debug( - "_prune_cache. after len: %d", - len(self._state_cache.keys()) - ) diff --git a/synapse/util/expiringcache.py b/synapse/util/expiringcache.py new file mode 100644 index 0000000000..fb5b27ad10 --- /dev/null +++ b/synapse/util/expiringcache.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + + +logger = logging.getLogger(__name__) + + +class ExpiringCache(object): + def __init__(self, cache_name, clock, max_len=0, expiry_ms=0, + reset_expiry_on_get=False): + """ + Args: + cache_name (str): Name of this cache, used for logging. + clock (Clock) + max_len (int): Max size of dict. If the dict grows larger than this + then the oldest items get automatically evicted. Default is 0, + which indicates there is no max limit. + expiry_ms (int): How long before an item is evicted from the cache + in milliseconds. Default is 0, indicating items never get + evicted based on time. + reset_expiry_on_get (bool): If true, will reset the expiry time for + an item on access. Defaults to False. + + """ + self._cache_name = cache_name + + self._clock = clock + + self._max_len = max_len + self._expiry_ms = expiry_ms + + self._reset_expiry_on_get = reset_expiry_on_get + + self._cache = {} + + def start(self): + if not self._expiry_ms: + # Don't bother starting the loop if things never expire + return + + def f(): + self._prune_cache() + + self._clock.looping_call(f, self._expiry_ms) + + def __setitem__(self, key, value): + now = self._clock.time_msec() + self._cache[key] = _CacheEntry(now, value) + + # Evict if there are now too many items + if self._max_len and len(self._cache.keys()) > self._max_len: + sorted_entries = sorted( + self._cache.items(), + key=lambda k, v: v.time, + ) + + for k, _ in sorted_entries[self._max_len:]: + self._cache.pop(k) + + def __getitem__(self, key): + entry = self._cache[key] + + if self._reset_expiry_on_get: + entry.time = self._clock.time_msec() + + return entry.value + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def _purge_cache(self): + if not self._expiry_ms: + # zero expiry time means don't expire. This should never get called + # since we have this check in start too. + return + begin_length = len(self._cache) + + now = self._clock.time_msec() + + keys_to_delete = set() + + for key, cache_entry in self._cache.items(): + if now - cache_entry.time > self._expiry_ms: + keys_to_delete.add(key) + + for k in keys_to_delete: + self._cache.pop(k) + + logger.debug( + "[%s] _prune_cache before: %d, after len: %d", + self._cache_name, begin_length, len(self._cache.keys()) + ) + + +class _CacheEntry(object): + def __init__(self, time, value): + self.time = time + self.value = value -- cgit 1.4.1 From baa5b9a97582d4b3c825be1225aba7863230c047 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Feb 2015 18:02:39 +0000 Subject: Cache results of get_pdu. --- synapse/federation/federation_client.py | 42 ++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 70c9a6f46b..83b4947b99 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -19,7 +19,8 @@ from twisted.internet import defer from .federation_base import FederationBase from .units import Edu -from synapse.api.errors import CodeMessageException +from synapse.api.errors import CodeMessageException, SynapseError +from synapse.util.expiringcache import ExpiringCache from synapse.util.logutils import log_function from synapse.events import FrozenEvent @@ -30,6 +31,20 @@ logger = logging.getLogger(__name__) class FederationClient(FederationBase): + def __init__(self): + self._fail_fetch_pdu_cache = None + + def start_pdu_fail_cache(self): + self._fail_fetch_pdu_cache = ExpiringCache( + cache_name="get_pdu_cache", + clock=self._clock, + max_len=1000, + expiry_ms=120*1000, + reset_expiry_on_get=False, + ) + + self._fail_fetch_pdu_cache.start() + @log_function def send_pdu(self, pdu, destinations): """Informs the replication layer about a new PDU generated within the @@ -160,6 +175,11 @@ class FederationClient(FederationBase): # TODO: Rate limit the number of times we try and get the same event. + if self._fail_fetch_pdu_cache: + e = self._fail_fetch_pdu_cache.get(event_id) + if e: + defer.returnValue(e) + pdu = None for destination in destinations: try: @@ -181,8 +201,21 @@ class FederationClient(FederationBase): pdu = yield self._check_sigs_and_hash(pdu) break - except CodeMessageException: - raise + except SynapseError: + logger.info( + "Failed to get PDU %s from %s because %s", + event_id, destination, e, + ) + continue + except CodeMessageException as e: + if 400 <= e.code < 500: + raise + + logger.info( + "Failed to get PDU %s from %s because %s", + event_id, destination, e, + ) + continue except Exception as e: logger.info( "Failed to get PDU %s from %s because %s", @@ -190,6 +223,9 @@ class FederationClient(FederationBase): ) continue + if self._fail_fetch_pdu_cache is not None: + self._fail_fetch_pdu_cache[event_id] = pdu + defer.returnValue(pdu) @defer.inlineCallbacks -- cgit 1.4.1 From 72a4de2ce627528a13bda480403344ffde6275d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 10:03:23 +0000 Subject: Use consumeErrors=True on all DeferredLists. This is so that the DeferredLists actually consume the error instead of propogating down the non-existent errback chain. This should reduce the number of unhandled errors we are seeing. --- synapse/federation/federation_server.py | 2 +- synapse/federation/transaction_queue.py | 2 +- synapse/handlers/presence.py | 8 ++++---- synapse/notifier.py | 6 ++++-- synapse/storage/roommember.py | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 9f5c98694c..e94d0411b4 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -124,7 +124,7 @@ class FederationServer(FederationBase): edu.content ) - results = yield defer.DeferredList(dl) + results = yield defer.DeferredList(dl, consumeErrors=True) ret = [] for r in results: diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 731019ad9f..bb20f2ebab 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -98,7 +98,7 @@ class TransactionQueue(object): deferreds.append(deferred) - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) # NO inlineCallbacks def enqueue_edu(self, edu): diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 59287010ed..8ef248ecf2 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -492,7 +492,7 @@ class PresenceHandler(BaseHandler): user, domain, remoteusers )) - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) def _start_polling_local(self, user, target_user): target_localpart = target_user.localpart @@ -548,7 +548,7 @@ class PresenceHandler(BaseHandler): self._stop_polling_remote(user, domain, remoteusers) ) - return defer.DeferredList(deferreds) + return defer.DeferredList(deferreds, consumeErrors=True) def _stop_polling_local(self, user, target_user): for localpart in self._local_pushmap.keys(): @@ -729,7 +729,7 @@ class PresenceHandler(BaseHandler): del self._remote_sendmap[user] with PreserveLoggingContext(): - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) @defer.inlineCallbacks def push_update_to_local_and_remote(self, observed_user, statuscache, @@ -768,7 +768,7 @@ class PresenceHandler(BaseHandler): ) ) - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) defer.returnValue((localusers, remote_domains)) diff --git a/synapse/notifier.py b/synapse/notifier.py index e3b6ead620..f5a394596d 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -135,7 +135,8 @@ class Notifier(object): with PreserveLoggingContext(): yield defer.DeferredList( - [notify(l).addErrback(eb) for l in listeners] + [notify(l).addErrback(eb) for l in listeners], + consumeErrors=True, ) @defer.inlineCallbacks @@ -203,7 +204,8 @@ class Notifier(object): with PreserveLoggingContext(): yield defer.DeferredList( - [notify(l).addErrback(eb) for l in listeners] + [notify(l).addErrback(eb) for l in listeners], + consumeErrors=True, ) @defer.inlineCallbacks diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 779f9ce544..9bf608bc90 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -288,7 +288,7 @@ class RoomMemberStore(SQLBaseStore): deferreds = [self.get_rooms_for_user(u) for u in user_id_list] - results = yield defer.DeferredList(deferreds) + results = yield defer.DeferredList(deferreds, consumeErrors=True) # A list of sets of strings giving room IDs for each user room_id_lists = [set([r.room_id for r in result[1]]) for result in results] -- cgit 1.4.1 From ea1d6c16cd2b91ec2980eef31446990cf305b796 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 10:54:06 +0000 Subject: Don't write bytecode --- synapse/app/homeserver.py | 4 +++- synapse/app/synctl.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f5681fac20..18585e87f2 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys +sys.dont_write_bytecode = True + from synapse.storage import prepare_database, UpgradeDatabaseException from synapse.server import HomeServer @@ -48,7 +51,6 @@ import synapse import logging import os import re -import sys import sqlite3 import syweb diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 363c20f994..3a70a248dc 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -19,7 +19,7 @@ import os import subprocess import signal -SYNAPSE = ["python", "-m", "synapse.app.homeserver"] +SYNAPSE = ["python", "-B", "-m", "synapse.app.homeserver"] CONFIGFILE = "homeserver.yaml" PIDFILE = "homeserver.pid" -- cgit 1.4.1 From c2b1dbd84c27be8759b14502b124504cba4c9295 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 11:11:11 +0000 Subject: We do want to consumeError --- synapse/handlers/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index c69787005f..c2762f92c7 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -181,7 +181,7 @@ class TypingNotificationHandler(BaseHandler): }, )) - yield defer.DeferredList(deferreds, consumeErrors=False) + yield defer.DeferredList(deferreds, consumeErrors=True) @defer.inlineCallbacks def _recv_edu(self, origin, content): -- cgit 1.4.1 From 02bfa889de1990eb8dc47770834bef777252dc82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 13:08:27 +0000 Subject: Handle recieving failures in transactions --- synapse/federation/federation_server.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index e94d0411b4..34f5b17446 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -114,7 +114,15 @@ class FederationServer(FederationBase): with PreserveLoggingContext(): dl = [] for pdu in pdu_list: - dl.append(self._handle_new_pdu(transaction.origin, pdu)) + d = self._handle_new_pdu(transaction.origin, pdu) + + def handle_failure(failure): + failure.trap(FederationError) + self.enqueue_failure(failure.value, transaction.origin) + + d.addErrback(handle_failure) + + dl.append(d) if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: @@ -124,6 +132,9 @@ class FederationServer(FederationBase): edu.content ) + for failure in getattr(transaction, "failures", []): + logger.info("Got failure %r", failure) + results = yield defer.DeferredList(dl, consumeErrors=True) ret = [] -- cgit 1.4.1 From 47281f8fa4b3f529f84409367736483da66416ae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 11:51:22 +0000 Subject: Change some debug logging to info --- synapse/handlers/federation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3aecc1ca51..0eb2ff95ca 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -802,7 +802,7 @@ class FederationHandler(BaseHandler): missing_auth = event_auth_events - seen_events if missing_auth: - logger.debug("Missing auth: %s", missing_auth) + logger.info("Missing auth: %s", missing_auth) # If we don't have all the auth events, we need to get them. try: remote_auth_chain = yield self.replication_layer.get_event_auth( @@ -856,7 +856,7 @@ class FederationHandler(BaseHandler): if different_auth and not event.internal_metadata.is_outlier(): # Do auth conflict res. - logger.debug("Different auth: %s", different_auth) + logger.info("Different auth: %s", different_auth) different_events = yield defer.gatherResults( [ @@ -892,6 +892,8 @@ class FederationHandler(BaseHandler): context.state_group = None if different_auth and not event.internal_metadata.is_outlier(): + logger.info("Different auth after resolution: %s", different_auth) + # Only do auth resolution if we have something new to say. # We can't rove an auth failure. do_resolution = False -- cgit 1.4.1 From c82e26ad4ba80742adc68a7e8c52441d760e4746 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 13:24:13 +0000 Subject: Actually respond with JSON to incoming transaction --- synapse/federation/federation_server.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 34f5b17446..c48a41b5d5 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -147,6 +147,8 @@ class FederationServer(FederationBase): logger.debug("Returning: %s", str(ret)) + response = ret + yield self.transaction_actions.set_response( transaction, 200, response -- cgit 1.4.1 From 659ead082ff11842fea803e44d75667e0ca38d71 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 13:58:52 +0000 Subject: Format the response of transaction request in a nicer way --- synapse/federation/federation_server.py | 19 +++++++++++++++---- synapse/federation/transaction_queue.py | 22 ++++++++++++++++++++-- 2 files changed, 35 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index c48a41b5d5..d1ec0b9eac 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -118,7 +118,7 @@ class FederationServer(FederationBase): def handle_failure(failure): failure.trap(FederationError) - self.enqueue_failure(failure.value, transaction.origin) + self.send_failure(failure.value, transaction.origin) d.addErrback(handle_failure) @@ -132,7 +132,7 @@ class FederationServer(FederationBase): edu.content ) - for failure in getattr(transaction, "failures", []): + for failure in getattr(transaction, "pdu_failures", []): logger.info("Got failure %r", failure) results = yield defer.DeferredList(dl, consumeErrors=True) @@ -143,11 +143,15 @@ class FederationServer(FederationBase): ret.append({}) else: logger.exception(r[1]) - ret.append({"error": str(r[1])}) + ret.append({"error": str(r[1].value)}) logger.debug("Returning: %s", str(ret)) - response = ret + response = { + "pdus": dict(zip( + (p.event_id for p in pdu_list), ret + )), + } yield self.transaction_actions.set_response( transaction, @@ -358,6 +362,13 @@ class FederationServer(FederationBase): affected=pdu.event_id, ) + raise FederationError( + "ERROR", + 403, + "Forbidden", + affected=pdu.event_id, + ) + state = None auth_chain = [] diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index bb20f2ebab..6faaa066fb 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -91,7 +91,7 @@ class TransactionQueue(object): if not deferred.called: deferred.errback(failure) else: - logger.warn("Failed to send pdu", failure) + logger.warn("Failed to send pdu", failure.value) with PreserveLoggingContext(): self._attempt_new_transaction(destination).addErrback(eb) @@ -116,7 +116,7 @@ class TransactionQueue(object): if not deferred.called: deferred.errback(failure) else: - logger.warn("Failed to send edu", failure) + logger.warn("Failed to send edu", failure.value) with PreserveLoggingContext(): self._attempt_new_transaction(destination).addErrback(eb) @@ -133,6 +133,15 @@ class TransactionQueue(object): (failure, deferred) ) + def eb(failure): + if not deferred.called: + deferred.errback(failure) + else: + logger.warn("Failed to send failure", failure.value) + + with PreserveLoggingContext(): + self._attempt_new_transaction(destination).addErrback(eb) + yield deferred @defer.inlineCallbacks @@ -249,6 +258,15 @@ class TransactionQueue(object): transaction, json_data_cb ) code = 200 + + if response: + for e_id, r in getattr(response, "pdus", {}).items(): + if "error" in r: + logger.warn( + "Transaction returned error for %s: %s", + e_id, r, + ) + except HttpResponseException as e: code = e.code response = e.response -- cgit 1.4.1 From 4de93001bf6a7d8e770b990ea3546237b2569609 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:12:06 +0000 Subject: Make matrixfederationclient log more nicely --- synapse/http/matrixfederationclient.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1927948001..764b151d9a 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -16,6 +16,7 @@ from twisted.internet import defer, reactor, protocol from twisted.internet.error import DNSLookupError +from twisted.python.failure import Failure from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone @@ -146,14 +147,22 @@ class MatrixFederationHttpClient(object): ) raise SynapseError(400, "Domain specified not found.") + if hasattr(e, "reasons"): + reasons = ", ".join( + f.value.message + for f in e.reasons + ) + else: + reasons = e.message + logger.warn( - "Sending request failed to %s: %s %s : %s", + "Sending request failed to %s: %s %s: %s - %s", destination, method, url_bytes, - e + type(e). __name__, + reasons, ) - _print_ex(e) if retries_left: yield sleep(2 ** (5 - retries_left)) @@ -447,14 +456,6 @@ def _readBodyToFile(response, stream, max_size): return d -def _print_ex(e): - if hasattr(e, "reasons") and e.reasons: - for ex in e.reasons: - _print_ex(ex) - else: - logger.warn(e) - - class _JsonProducer(object): """ Used by the twisted http client to create the HTTP body from json """ -- cgit 1.4.1 From 472734a8cc19d4e4b9f5e311b59f58a37142f8a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:13:50 +0000 Subject: Consume errors in time_bound_deferred --- synapse/util/__init__.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse') diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index e77eba90ad..79109d0b19 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -99,8 +99,6 @@ class Clock(object): except: pass - return res - given_deferred.addCallbacks(callback=sucess, errback=err) timer = self.call_later(time_out, timed_out_fn) -- cgit 1.4.1 From 0647e27a414e5a86cab57bba65931376e855c289 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:19:54 +0000 Subject: Remove unused import --- synapse/http/matrixfederationclient.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 764b151d9a..454c3d4ab1 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -16,7 +16,6 @@ from twisted.internet import defer, reactor, protocol from twisted.internet.error import DNSLookupError -from twisted.python.failure import Failure from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone -- cgit 1.4.1 From 676e8ee78ae3f51cbc0113c8d810d4bc1e81cdab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:22:45 +0000 Subject: Remove debug raise --- synapse/federation/federation_server.py | 7 ------- 1 file changed, 7 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 679fb141e7..22b9663831 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -361,13 +361,6 @@ class FederationServer(FederationBase): affected=pdu.event_id, ) - raise FederationError( - "ERROR", - 403, - "Forbidden", - affected=pdu.event_id, - ) - state = None auth_chain = [] -- cgit 1.4.1 From 8b919c00f30c03ddae257f0129f58f2d0285723c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:44:01 +0000 Subject: Start the get_pdu cache --- synapse/app/homeserver.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 27b478a1c3..7565d94449 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -275,6 +275,7 @@ def setup(): hs.get_pusherpool().start() hs.get_state_handler().start_caching() hs.get_datastore().start_profiling() + hs.get_replication_layer().start_pdu_fail_cache() if config.daemonize: print config.pid_file -- cgit 1.4.1 From e7e20417ca468c1afe2e27162b9790f860d2da51 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:44:26 +0000 Subject: ExpiringCache: purge every 1/2 interval --- synapse/util/expiringcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/util/expiringcache.py b/synapse/util/expiringcache.py index fb5b27ad10..3c8409b164 100644 --- a/synapse/util/expiringcache.py +++ b/synapse/util/expiringcache.py @@ -55,7 +55,7 @@ class ExpiringCache(object): def f(): self._prune_cache() - self._clock.looping_call(f, self._expiry_ms) + self._clock.looping_call(f, self._expiry_ms/2) def __setitem__(self, key, value): now = self._clock.time_msec() -- cgit 1.4.1 From 964bb43fbe15105c58f9550f376fa76709734cd8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:44:41 +0000 Subject: Fix typo in function name --- synapse/util/expiringcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/util/expiringcache.py b/synapse/util/expiringcache.py index 3c8409b164..1c7859297a 100644 --- a/synapse/util/expiringcache.py +++ b/synapse/util/expiringcache.py @@ -85,7 +85,7 @@ class ExpiringCache(object): except KeyError: return default - def _purge_cache(self): + def _prune_cache(self): if not self._expiry_ms: # zero expiry time means don't expire. This should never get called # since we have this check in start too. -- cgit 1.4.1 From 1a989c436cda4926f11b04b4f26d83e7d3ce9ef5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:45:55 +0000 Subject: Bump schema version --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/v13.sql | 34 ++++++++++++++++++++++++++++++++++ synapse/storage/schema/delta/v14.sql | 34 ---------------------------------- 3 files changed, 35 insertions(+), 35 deletions(-) create mode 100644 synapse/storage/schema/delta/v13.sql delete mode 100644 synapse/storage/schema/delta/v14.sql (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c6e96b842f..ec701014a9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -74,7 +74,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 12 +SCHEMA_VERSION = 13 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v13.sql b/synapse/storage/schema/delta/v13.sql new file mode 100644 index 0000000000..e491ad5aec --- /dev/null +++ b/synapse/storage/schema/delta/v13.sql @@ -0,0 +1,34 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT, + token TEXT, + hs_token TEXT, + sender TEXT, + UNIQUE(token) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id INTEGER PRIMARY KEY AUTOINCREMENT, + as_id INTEGER NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex TEXT, + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + + + diff --git a/synapse/storage/schema/delta/v14.sql b/synapse/storage/schema/delta/v14.sql deleted file mode 100644 index e491ad5aec..0000000000 --- a/synapse/storage/schema/delta/v14.sql +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS application_services( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT, - token TEXT, - hs_token TEXT, - sender TEXT, - UNIQUE(token) ON CONFLICT ROLLBACK -); - -CREATE TABLE IF NOT EXISTS application_services_regex( - id INTEGER PRIMARY KEY AUTOINCREMENT, - as_id INTEGER NOT NULL, - namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex TEXT, - FOREIGN KEY(as_id) REFERENCES application_services(id) -); - - - -- cgit 1.4.1 From 5025305fb250c358236584cae06bf0855f02b0a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:57:42 +0000 Subject: Rate limit retries when fetching server keys. --- synapse/crypto/keyring.py | 126 ++++++++++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 55 deletions(-) (limited to 'synapse') diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 3fb99f7125..3250cff595 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -87,69 +87,85 @@ class Keyring(object): return # Try to fetch the key from the remote server. - # TODO(markjh): Ratelimit requests to a given server. - (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory + retry_last_ts, retry_interval = (0, 0) + retry_timings = yield self.store.get_destination_retry_timings( + server_name ) + if retry_timings: + retry_last_ts, retry_interval = ( + retry_timings.retry_last_ts, retry_timings.retry_interval + ) + if retry_last_ts + retry_interval > int(self.clock.time_msec()): + logger.info("%s not ready for retry", server_name) + raise ValueError("No verification key found for given key ids") - # Check the response. - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) - - if ("signatures" not in response - or server_name not in response["signatures"]): - raise ValueError("Key response not signed by remote server") - - if "tls_certificate" not in response: - raise ValueError("Key response missing TLS certificate") + try: + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory + ) - tls_certificate_b64 = response["tls_certificate"] + # Check the response. - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate + ) - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_keys[key_id] = verify_key + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") + + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") + + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + yield self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise ValueError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] + for key_id, key in verify_keys.items(): + yield self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key ) - # Cache the result in the datastore. - - time_now_ms = self.clock.time_msec() - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) - - for key_id, key in verify_keys.items(): - yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key - ) + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return + raise ValueError("No verification key found for given key ids") - raise ValueError("No verification key found for given key ids") + except: + self.set_retrying(server_name, retry_interval) + raise -- cgit 1.4.1 From 2b8f1a956c6a1d767a3b60e84e7d0afe5857fb0d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 17:20:56 +0000 Subject: Add per server retry limiting. Factor out the pre destination retry logic from TransactionQueue so it can be reused in both get_pdu and crypto.keyring --- synapse/crypto/keyring.py | 22 ++--- synapse/federation/federation_client.py | 36 ++++--- synapse/federation/transaction_queue.py | 161 +++++++++++++------------------- synapse/util/retryutils.py | 108 +++++++++++++++++++++ 4 files changed, 205 insertions(+), 122 deletions(-) create mode 100644 synapse/util/retryutils.py (limited to 'synapse') diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 3250cff595..ea00c830c0 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -22,6 +22,8 @@ from syutil.crypto.signing_key import ( from syutil.base64util import decode_base64, encode_base64 from synapse.api.errors import SynapseError, Codes +from synapse.util.retryutils import get_retry_limiter + from OpenSSL import crypto import logging @@ -88,19 +90,13 @@ class Keyring(object): # Try to fetch the key from the remote server. - retry_last_ts, retry_interval = (0, 0) - retry_timings = yield self.store.get_destination_retry_timings( - server_name + limiter = yield get_retry_limiter( + server_name, + self.clock, + self.store, ) - if retry_timings: - retry_last_ts, retry_interval = ( - retry_timings.retry_last_ts, retry_timings.retry_interval - ) - if retry_last_ts + retry_interval > int(self.clock.time_msec()): - logger.info("%s not ready for retry", server_name) - raise ValueError("No verification key found for given key ids") - try: + with limiter: (response, tls_certificate) = yield fetch_server_key( server_name, self.hs.tls_context_factory ) @@ -165,7 +161,3 @@ class Keyring(object): return raise ValueError("No verification key found for given key ids") - - except: - self.set_retrying(server_name, retry_interval) - raise diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 70c9a6f46b..c5b0274c2f 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -23,6 +23,8 @@ from synapse.api.errors import CodeMessageException from synapse.util.logutils import log_function from synapse.events import FrozenEvent +from synapse.util.retryutils import get_retry_limiter, NotRetryingDestination + import logging @@ -163,24 +165,34 @@ class FederationClient(FederationBase): pdu = None for destination in destinations: try: - transaction_data = yield self.transport_layer.get_event( - destination, event_id + limiter = yield get_retry_limiter( + destination, + self._clock, + self.store, ) - logger.debug("transaction_data %r", transaction_data) + with limiter: + transaction_data = yield self.transport_layer.get_event( + destination, event_id + ) - pdu_list = [ - self.event_from_pdu_json(p, outlier=outlier) - for p in transaction_data["pdus"] - ] + logger.debug("transaction_data %r", transaction_data) - if pdu_list: - pdu = pdu_list[0] + pdu_list = [ + self.event_from_pdu_json(p, outlier=outlier) + for p in transaction_data["pdus"] + ] - # Check signatures are correct. - pdu = yield self._check_sigs_and_hash(pdu) + if pdu_list: + pdu = pdu_list[0] - break + # Check signatures are correct. + pdu = yield self._check_sigs_and_hash(pdu) + + break + except NotRetryingDestination as e: + logger.info(e.message) + continue except CodeMessageException: raise except Exception as e: diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index bb20f2ebab..7d02afe163 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -22,6 +22,9 @@ from .units import Transaction from synapse.api.errors import HttpResponseException from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext +from synapse.util.retryutils import ( + get_retry_limiter, NotRetryingDestination, +) import logging @@ -138,25 +141,6 @@ class TransactionQueue(object): @defer.inlineCallbacks @log_function def _attempt_new_transaction(self, destination): - - (retry_last_ts, retry_interval) = (0, 0) - retry_timings = yield self.store.get_destination_retry_timings( - destination - ) - if retry_timings: - (retry_last_ts, retry_interval) = ( - retry_timings.retry_last_ts, retry_timings.retry_interval - ) - if retry_last_ts + retry_interval > int(self._clock.time_msec()): - logger.info( - "TX [%s] not ready for retry yet - " - "dropping transaction for now", - destination, - ) - return - else: - logger.info("TX [%s] is ready for retry", destination) - if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending # request at which point pending_pdus_by_dest just keeps growing. @@ -204,77 +188,79 @@ class TransactionQueue(object): ] try: - self.pending_transactions[destination] = 1 - - logger.debug("TX [%s] Persisting transaction...", destination) - - transaction = Transaction.create_new( - origin_server_ts=int(self._clock.time_msec()), - transaction_id=str(self._next_txn_id), - origin=self.server_name, - destination=destination, - pdus=pdus, - edus=edus, - pdu_failures=failures, + limiter = yield get_retry_limiter( + destination, + self._clock, + self.store, ) - self._next_txn_id += 1 + with limiter: + self.pending_transactions[destination] = 1 - yield self.transaction_actions.prepare_to_send(transaction) + logger.debug("TX [%s] Persisting transaction...", destination) - logger.debug("TX [%s] Persisted transaction", destination) - logger.info( - "TX [%s] Sending transaction [%s]", - destination, - transaction.transaction_id, - ) + transaction = Transaction.create_new( + origin_server_ts=int(self._clock.time_msec()), + transaction_id=str(self._next_txn_id), + origin=self.server_name, + destination=destination, + pdus=pdus, + edus=edus, + pdu_failures=failures, + ) + + self._next_txn_id += 1 - # Actually send the transaction - - # FIXME (erikj): This is a bit of a hack to make the Pdu age - # keys work - def json_data_cb(): - data = transaction.get_dict() - now = int(self._clock.time_msec()) - if "pdus" in data: - for p in data["pdus"]: - if "age_ts" in p: - unsigned = p.setdefault("unsigned", {}) - unsigned["age"] = now - int(p["age_ts"]) - del p["age_ts"] - return data - - try: - response = yield self.transport_layer.send_transaction( - transaction, json_data_cb + yield self.transaction_actions.prepare_to_send(transaction) + + logger.debug("TX [%s] Persisted transaction", destination) + logger.info( + "TX [%s] Sending transaction [%s]", + destination, + transaction.transaction_id, ) - code = 200 - except HttpResponseException as e: - code = e.code - response = e.response - logger.info("TX [%s] got %d response", destination, code) + # Actually send the transaction + + # FIXME (erikj): This is a bit of a hack to make the Pdu age + # keys work + def json_data_cb(): + data = transaction.get_dict() + now = int(self._clock.time_msec()) + if "pdus" in data: + for p in data["pdus"]: + if "age_ts" in p: + unsigned = p.setdefault("unsigned", {}) + unsigned["age"] = now - int(p["age_ts"]) + del p["age_ts"] + return data + + try: + response = yield self.transport_layer.send_transaction( + transaction, json_data_cb + ) + code = 200 + except HttpResponseException as e: + code = e.code + response = e.response - logger.debug("TX [%s] Sent transaction", destination) - logger.debug("TX [%s] Marking as delivered...", destination) + logger.info("TX [%s] got %d response", destination, code) - yield self.transaction_actions.delivered( - transaction, code, response - ) + logger.debug("TX [%s] Sent transaction", destination) + logger.debug("TX [%s] Marking as delivered...", destination) + + yield self.transaction_actions.delivered( + transaction, code, response + ) + + logger.debug("TX [%s] Marked as delivered", destination) - logger.debug("TX [%s] Marked as delivered", destination) logger.debug("TX [%s] Yielding to callbacks...", destination) for deferred in deferreds: if code == 200: - if retry_last_ts: - # this host is alive! reset retry schedule - yield self.store.set_destination_retry_timings( - destination, 0, 0 - ) deferred.callback(None) else: - self.set_retrying(destination, retry_interval) deferred.errback(RuntimeError("Got status %d" % code)) # Ensures we don't continue until all callbacks on that @@ -285,6 +271,12 @@ class TransactionQueue(object): pass logger.debug("TX [%s] Yielded to callbacks", destination) + except NotRetryingDestination: + logger.info( + "TX [%s] not ready for retry yet - " + "dropping transaction for now", + destination, + ) except RuntimeError as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. @@ -302,8 +294,6 @@ class TransactionQueue(object): e, ) - self.set_retrying(destination, retry_interval) - for deferred in deferreds: if not deferred.called: deferred.errback(e) @@ -314,22 +304,3 @@ class TransactionQueue(object): # Check to see if there is anything else to send. self._attempt_new_transaction(destination) - - @defer.inlineCallbacks - def set_retrying(self, destination, retry_interval): - # track that this destination is having problems and we should - # give it a chance to recover before trying it again - - if retry_interval: - retry_interval *= 2 - # plateau at hourly retries for now - if retry_interval >= 60 * 60 * 1000: - retry_interval = 60 * 60 * 1000 - else: - retry_interval = 2000 # try again at first after 2 seconds - - yield self.store.set_destination_retry_timings( - destination, - int(self._clock.time_msec()), - retry_interval - ) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py new file mode 100644 index 0000000000..bba524ebd8 --- /dev/null +++ b/synapse/util/retryutils.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +import logging + + +logger = logging.getLogger(__name__) + + +class NotRetryingDestination(Exception): + def __init__(self, retry_last_ts, retry_interval, destination): + msg = "Not retrying server %s." % (destination,) + super(NotRetryingDestination, self).__init__(msg) + + self.retry_last_ts = retry_last_ts + self.retry_interval = retry_interval + self.destination = destination + + +@defer.inlineCallbacks +def get_retry_limiter(destination, clock, store, **kwargs): + retry_last_ts, retry_interval = (0, 0) + + retry_timings = yield store.get_destination_retry_timings( + destination + ) + + if retry_timings: + retry_last_ts, retry_interval = ( + retry_timings.retry_last_ts, retry_timings.retry_interval + ) + + now = int(clock.time_msec()) + + if retry_last_ts + retry_interval > now: + raise NotRetryingDestination( + retry_last_ts=retry_last_ts, + retry_interval=retry_interval, + destination=destination, + ) + + defer.returnValue( + RetryDestinationLimiter( + destination, + clock, + store, + retry_interval, + **kwargs + ) + ) + + +class RetryDestinationLimiter(object): + def __init__(self, destination, clock, store, retry_interval, + min_retry_interval=20000, max_retry_interval=60 * 60 * 1000, + multiplier_retry_interval=2): + self.clock = clock + self.store = store + self.destination = destination + + self.retry_interval = retry_interval + self.min_retry_interval = min_retry_interval + self.max_retry_interval = max_retry_interval + self.multiplier_retry_interval = multiplier_retry_interval + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + def err(self, failure): + logger.exception( + "Failed to store set_destination_retry_timings", + failure.value + ) + + if exc_type is None and exc_val is None and exc_tb is None: + # We connected successfully. + retry_last_ts = 0 + self.retry_interval = 0 + else: + # We couldn't connect. + if self.retry_interval: + self.retry_interval *= self.multiplier_retry_interval + + if self.retry_interval >= self.max_retry_interval: + self.retry_interval = self.max_retry_interval + else: + self.retry_interval = self.min_retry_interval + + retry_last_ts = int(self._clock.time_msec()), + + self.store.set_destination_retry_timings( + self.destination, retry_last_ts, self.retry_interval + ).addErrback(err) -- cgit 1.4.1 From 2c29ed3e847285973ab552b9617750e1ba6693e0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 17:22:24 +0000 Subject: Use absolute path when loading delta sql files --- synapse/storage/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index ec701014a9..d16e7b8fac 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -76,6 +76,8 @@ SCHEMAS = [ # database schema files, so the users will be informed on server restarts. SCHEMA_VERSION = 13 +dir_path = os.path.abspath(os.path.dirname(__file__)) + class _RollbackButIsFineException(Exception): """ This exception is used to rollback a transaction without implying @@ -583,7 +585,6 @@ def schema_path(schema): A filesystem path pointing at a ".sql" file. """ - dir_path = os.path.dirname(__file__) schemaPath = os.path.join(dir_path, "schema", schema + ".sql") return schemaPath -- cgit 1.4.1 From f91263b1e03a6e30480af1ece357c27439a8c6b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 17:37:51 +0000 Subject: Remove spurious self --- synapse/util/retryutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index bba524ebd8..9ab119f29e 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -81,7 +81,7 @@ class RetryDestinationLimiter(object): pass def __exit__(self, exc_type, exc_val, exc_tb): - def err(self, failure): + def err(failure): logger.exception( "Failed to store set_destination_retry_timings", failure.value -- cgit 1.4.1 From c8436b38a08c29fd7027e040564678ec8a635a14 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 17:38:38 +0000 Subject: Only update destination_retry_timings if we have succeeded when retrying --- synapse/util/retryutils.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'synapse') diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 9ab119f29e..888b7ef2e9 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -89,6 +89,9 @@ class RetryDestinationLimiter(object): if exc_type is None and exc_val is None and exc_tb is None: # We connected successfully. + if not self.retry_interval: + return + retry_last_ts = 0 self.retry_interval = 0 else: -- cgit 1.4.1 From 9371019133bf16cec163d58fe69aa701c8ca5305 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 18:13:34 +0000 Subject: Try to only back off if we think we failed to connect to the remote --- synapse/crypto/keyring.py | 108 ++++++++++++++++---------------- synapse/federation/transaction_queue.py | 66 +++++++++---------- synapse/util/retryutils.py | 10 ++- 3 files changed, 95 insertions(+), 89 deletions(-) (limited to 'synapse') diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index ea00c830c0..828aced44a 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -101,63 +101,63 @@ class Keyring(object): server_name, self.hs.tls_context_factory ) - # Check the response. + # Check the response. - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate + ) - if ("signatures" not in response - or server_name not in response["signatures"]): - raise ValueError("Key response not signed by remote server") - - if "tls_certificate" not in response: - raise ValueError("Key response missing TLS certificate") - - tls_certificate_b64 = response["tls_certificate"] - - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") - - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_keys[key_id] = verify_key - - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise ValueError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] - ) - - # Cache the result in the datastore. - - time_now_ms = self.clock.time_msec() - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") + + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") - for key_id, key in verify_keys.items(): - yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] ) - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + yield self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + yield self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return - raise ValueError("No verification key found for given key ids") + raise ValueError("No verification key found for given key ids") diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 7d02afe163..dc0f30cb64 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -167,15 +167,6 @@ class TransactionQueue(object): logger.info("TX [%s] Nothing to send", destination) return - logger.debug( - "TX [%s] Attempting new transaction" - " (pdus: %d, edus: %d, failures: %d)", - destination, - len(pending_pdus), - len(pending_edus), - len(pending_failures) - ) - # Sort based on the order field pending_pdus.sort(key=lambda t: t[2]) @@ -194,32 +185,41 @@ class TransactionQueue(object): self.store, ) - with limiter: - self.pending_transactions[destination] = 1 + logger.debug( + "TX [%s] Attempting new transaction" + " (pdus: %d, edus: %d, failures: %d)", + destination, + len(pending_pdus), + len(pending_edus), + len(pending_failures) + ) + + self.pending_transactions[destination] = 1 - logger.debug("TX [%s] Persisting transaction...", destination) + logger.debug("TX [%s] Persisting transaction...", destination) - transaction = Transaction.create_new( - origin_server_ts=int(self._clock.time_msec()), - transaction_id=str(self._next_txn_id), - origin=self.server_name, - destination=destination, - pdus=pdus, - edus=edus, - pdu_failures=failures, - ) + transaction = Transaction.create_new( + origin_server_ts=int(self._clock.time_msec()), + transaction_id=str(self._next_txn_id), + origin=self.server_name, + destination=destination, + pdus=pdus, + edus=edus, + pdu_failures=failures, + ) - self._next_txn_id += 1 + self._next_txn_id += 1 - yield self.transaction_actions.prepare_to_send(transaction) + yield self.transaction_actions.prepare_to_send(transaction) - logger.debug("TX [%s] Persisted transaction", destination) - logger.info( - "TX [%s] Sending transaction [%s]", - destination, - transaction.transaction_id, - ) + logger.debug("TX [%s] Persisted transaction", destination) + logger.info( + "TX [%s] Sending transaction [%s]", + destination, + transaction.transaction_id, + ) + with limiter: # Actually send the transaction # FIXME (erikj): This is a bit of a hack to make the Pdu age @@ -249,11 +249,11 @@ class TransactionQueue(object): logger.debug("TX [%s] Sent transaction", destination) logger.debug("TX [%s] Marking as delivered...", destination) - yield self.transaction_actions.delivered( - transaction, code, response - ) + yield self.transaction_actions.delivered( + transaction, code, response + ) - logger.debug("TX [%s] Marked as delivered", destination) + logger.debug("TX [%s] Marked as delivered", destination) logger.debug("TX [%s] Yielding to callbacks...", destination) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 888b7ef2e9..d190100c8c 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -15,6 +15,8 @@ from twisted.internet import defer +from synapse.api.errors import CodeMessageException + import logging @@ -67,7 +69,7 @@ def get_retry_limiter(destination, clock, store, **kwargs): class RetryDestinationLimiter(object): def __init__(self, destination, clock, store, retry_interval, min_retry_interval=20000, max_retry_interval=60 * 60 * 1000, - multiplier_retry_interval=2): + multiplier_retry_interval=2,): self.clock = clock self.store = store self.destination = destination @@ -87,7 +89,11 @@ class RetryDestinationLimiter(object): failure.value ) - if exc_type is None and exc_val is None and exc_tb is None: + valid_err_code = False + if exc_type is CodeMessageException: + valid_err_code = 0 <= exc_val.code < 500 + + if exc_type is None or valid_err_code: # We connected successfully. if not self.retry_interval: return -- cgit 1.4.1 From d77912ff4483fa63308eeefb70da93b8c327b740 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:09:54 +0000 Subject: Docs. --- synapse/util/retryutils.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index d190100c8c..08285e811d 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -35,6 +35,23 @@ class NotRetryingDestination(Exception): @defer.inlineCallbacks def get_retry_limiter(destination, clock, store, **kwargs): + """For a given destination check if we have previously failed to + send a request there and are waiting before retrying the destination. + If we are not ready to retry the destination, this will raise a + NotRetryingDestination exception. Otherwise, will return a Context Manager + that will mark the destination as down if an exception is thrown (excluding + CodeMessageException with code < 500) + + Example usage: + + try: + limiter = yield get_retry_limiter(destination, clock, store) + with limiter: + response = yield do_request() + except NotRetryingDestination: + # We aren't ready to retry that destination. + raise + """ retry_last_ts, retry_interval = (0, 0) retry_timings = yield store.get_destination_retry_timings( @@ -68,8 +85,23 @@ def get_retry_limiter(destination, clock, store, **kwargs): class RetryDestinationLimiter(object): def __init__(self, destination, clock, store, retry_interval, - min_retry_interval=20000, max_retry_interval=60 * 60 * 1000, + min_retry_interval=5000, max_retry_interval=60 * 60 * 1000, multiplier_retry_interval=2,): + """ + Args: + destination (str) + clock (Clock) + store (DataStore) + retry_interval (int): The next retry interval taken from the + database in milliseconds, or zero if the last request was + successful. + min_retry_interval (int): The minimum retry interval to use after + a failed request, in milliseconds. + max_retry_interval (int): The maximum retry interval to use after + a failed request, in milliseconds. + multiplier_retry_interval (int): The multiplier to use to increase + the retry interval after a failed request. + """ self.clock = clock self.store = store self.destination = destination -- cgit 1.4.1 From 4fd176a41db9f3b9073db172e20c8ede40b5f5f4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:11:24 +0000 Subject: More docs --- synapse/util/retryutils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 08285e811d..366f7c48a7 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -87,7 +87,11 @@ class RetryDestinationLimiter(object): def __init__(self, destination, clock, store, retry_interval, min_retry_interval=5000, max_retry_interval=60 * 60 * 1000, multiplier_retry_interval=2,): - """ + """Marks the destination as "down" if an exception is thrown in the + context, except for CodeMessageException with code < 500. + + If no exception is raised, marks the destination as "up". + Args: destination (str) clock (Clock) -- cgit 1.4.1 From ec847059f3e9b9b5de62aa2f7ad2366c4e883fac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:14:10 +0000 Subject: Rename _fail_fetch_pdu_cache to _get_pdu_cache --- synapse/app/homeserver.py | 2 +- synapse/federation/federation_client.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 7565d94449..7be82d0576 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -275,7 +275,7 @@ def setup(): hs.get_pusherpool().start() hs.get_state_handler().start_caching() hs.get_datastore().start_profiling() - hs.get_replication_layer().start_pdu_fail_cache() + hs.get_replication_layer().start_get_pdu_cache() if config.daemonize: print config.pid_file diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 83b4947b99..6042e366bd 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -32,10 +32,10 @@ logger = logging.getLogger(__name__) class FederationClient(FederationBase): def __init__(self): - self._fail_fetch_pdu_cache = None + self._get_pdu_cache = None - def start_pdu_fail_cache(self): - self._fail_fetch_pdu_cache = ExpiringCache( + def start_get_pdu_cache(self): + self._get_pdu_cache = ExpiringCache( cache_name="get_pdu_cache", clock=self._clock, max_len=1000, @@ -43,7 +43,7 @@ class FederationClient(FederationBase): reset_expiry_on_get=False, ) - self._fail_fetch_pdu_cache.start() + self._get_pdu_cache.start() @log_function def send_pdu(self, pdu, destinations): @@ -175,8 +175,8 @@ class FederationClient(FederationBase): # TODO: Rate limit the number of times we try and get the same event. - if self._fail_fetch_pdu_cache: - e = self._fail_fetch_pdu_cache.get(event_id) + if self._get_pdu_cache: + e = self._get_pdu_cache.get(event_id) if e: defer.returnValue(e) @@ -223,8 +223,8 @@ class FederationClient(FederationBase): ) continue - if self._fail_fetch_pdu_cache is not None: - self._fail_fetch_pdu_cache[event_id] = pdu + if self._get_pdu_cache is not None: + self._get_pdu_cache[event_id] = pdu defer.returnValue(pdu) -- cgit 1.4.1 From 859fbd4423c57a0d64dde5322dde46120692f64b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:39:14 +0000 Subject: s/self._clock/self.clock/ --- synapse/util/retryutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 366f7c48a7..ac30545079 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -146,7 +146,7 @@ class RetryDestinationLimiter(object): else: self.retry_interval = self.min_retry_interval - retry_last_ts = int(self._clock.time_msec()), + retry_last_ts = int(self.clock.time_msec()), self.store.set_destination_retry_timings( self.destination, retry_last_ts, self.retry_interval -- cgit 1.4.1 From 0db52d43fa41526b83d0c70141c43bfd89388820 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:41:46 +0000 Subject: strings.join() expects iterable of strings --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 454c3d4ab1..b5e201b5f1 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -148,7 +148,7 @@ class MatrixFederationHttpClient(object): if hasattr(e, "reasons"): reasons = ", ".join( - f.value.message + str(f.value.message) for f in e.reasons ) else: -- cgit 1.4.1 From e482541e1d77cb179dafa44037ee1e04003501ab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:44:22 +0000 Subject: Fix pyflakes --- synapse/federation/transaction_queue.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index a77201ad49..ae04774c76 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -266,7 +266,6 @@ class TransactionQueue(object): logger.debug("TX [%s] Sent transaction", destination) logger.debug("TX [%s] Marking as delivered...", destination) - yield self.transaction_actions.delivered( transaction, code, response ) -- cgit 1.4.1 From 5e2447146982a30d0d4ea1334297cc0bfb32c3c5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:50:10 +0000 Subject: Fix up ResponseNeverReceived to str --- synapse/http/matrixfederationclient.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b5e201b5f1..304efcdd56 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -146,21 +146,13 @@ class MatrixFederationHttpClient(object): ) raise SynapseError(400, "Domain specified not found.") - if hasattr(e, "reasons"): - reasons = ", ".join( - str(f.value.message) - for f in e.reasons - ) - else: - reasons = e.message - logger.warn( "Sending request failed to %s: %s %s: %s - %s", destination, method, url_bytes, type(e). __name__, - reasons, + _flatten_response_never_received(e), ) if retries_left: @@ -474,3 +466,13 @@ class _JsonProducer(object): def stopProducing(self): pass + + +def _flatten_response_never_received(e): + if hasattr(e, "reasons"): + return ", ".join( + _flatten_response_never_received(f.value) + for f in e.reasons + ) + else: + return "%s: %s" % (type(e), e.message,) -- cgit 1.4.1 From 65ca713ff53794bd517a85de7df063e631fb255a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:51:32 +0000 Subject: Add .__name__ after type(e) --- synapse/http/matrixfederationclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 304efcdd56..49a4c7d9d3 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -151,7 +151,7 @@ class MatrixFederationHttpClient(object): destination, method, url_bytes, - type(e). __name__, + type(e).__name__, _flatten_response_never_received(e), ) @@ -475,4 +475,4 @@ def _flatten_response_never_received(e): for f in e.reasons ) else: - return "%s: %s" % (type(e), e.message,) + return "%s: %s" % (type(e).__name__, e.message,) -- cgit 1.4.1 From cedad8fbd688eef70da4d758987bb39f46fe84bf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:54:34 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 792b8089f6..a804a25748 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0f" +__version__ = "0.7.1" -- cgit 1.4.1 From 7e9d59f3b4628981d5e53ac0cc71325ad896287c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:58:13 +0000 Subject: Don't convert DNSLookupError to a 4xx SynapseError --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 49a4c7d9d3..fe57363388 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -144,7 +144,7 @@ class MatrixFederationHttpClient(object): destination, e ) - raise SynapseError(400, "Domain specified not found.") + raise logger.warn( "Sending request failed to %s: %s %s: %s - %s", -- cgit 1.4.1 From 87e9aeb9143a3dab2dcac24797c822dedde845f1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Feb 2015 10:59:30 +0000 Subject: Move pynacl to the top of the depedency link list so that it is installed before syutil --- synapse/python_dependencies.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ec78fc3627..a30a570552 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -24,6 +24,11 @@ def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) DEPENDENCY_LINKS = [ + github_link( + project="pyca/pynacl", + version="d4d3175589b892f6ea7c22f466e0e223853516fa", + egg="pynacl-0.3.0", + ) github_link( project="matrix-org/syutil", version="v0.0.3", @@ -34,11 +39,6 @@ DEPENDENCY_LINKS = [ version="v0.6.2", egg="matrix_angular_sdk-0.6.2", ), - github_link( - project="pyca/pynacl", - version="d4d3175589b892f6ea7c22f466e0e223853516fa", - egg="pynacl-0.3.0", - ) ] -- cgit 1.4.1 From 5806d52423a97e219b710ca638866e0763fa416e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Feb 2015 11:01:37 +0000 Subject: Fix syntax --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a30a570552..9464b4fb62 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -28,7 +28,7 @@ DEPENDENCY_LINKS = [ project="pyca/pynacl", version="d4d3175589b892f6ea7c22f466e0e223853516fa", egg="pynacl-0.3.0", - ) + ), github_link( project="matrix-org/syutil", version="v0.0.3", -- cgit 1.4.1 From b17bd31da0574cf335f90763dee2708af40ac87f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 11:17:26 +0000 Subject: Temporarily add a run_on_reactor() call --- synapse/util/retryutils.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'synapse') diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index ac30545079..ee1194f1de 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -17,6 +17,8 @@ from twisted.internet import defer from synapse.api.errors import CodeMessageException +from synapse.util.async import run_on_reactor + import logging @@ -52,6 +54,7 @@ def get_retry_limiter(destination, clock, store, **kwargs): # We aren't ready to retry that destination. raise """ + yield run_on_reactor() retry_last_ts, retry_interval = (0, 0) retry_timings = yield store.get_destination_retry_timings( -- cgit 1.4.1 From 36e144091b3be995754f7100a9c5322b224f2692 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 11:25:20 +0000 Subject: Remove spurious comma. Remove temp run_on_reactor --- synapse/util/retryutils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index ee1194f1de..ab58eb528b 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -54,7 +54,6 @@ def get_retry_limiter(destination, clock, store, **kwargs): # We aren't ready to retry that destination. raise """ - yield run_on_reactor() retry_last_ts, retry_interval = (0, 0) retry_timings = yield store.get_destination_retry_timings( @@ -149,7 +148,7 @@ class RetryDestinationLimiter(object): else: self.retry_interval = self.min_retry_interval - retry_last_ts = int(self.clock.time_msec()), + retry_last_ts = int(self.clock.time_msec()) self.store.set_destination_retry_timings( self.destination, retry_last_ts, self.retry_interval -- cgit 1.4.1 From 47d3ff4cf8ebceacc680c9e5b414150c6edbf85b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 11:30:37 +0000 Subject: Don't send failure to self --- synapse/federation/transaction_queue.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'synapse') diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index ae04774c76..f0041d4d12 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -128,6 +128,9 @@ class TransactionQueue(object): @defer.inlineCallbacks def enqueue_failure(self, failure, destination): + if destination == self.server_name: + return + deferred = defer.Deferred() self.pending_failures_by_dest.setdefault( -- cgit 1.4.1 From b68e4a729ff91372a3a7269acc08c1f77584b0f4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 11:32:39 +0000 Subject: Discard destination 'localhost' --- synapse/federation/transaction_queue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index f0041d4d12..06944eefb9 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -107,7 +107,7 @@ class TransactionQueue(object): def enqueue_edu(self, edu): destination = edu.destination - if destination == self.server_name: + if destination == self.server_name or destination == "localhost": return deferred = defer.Deferred() @@ -128,7 +128,7 @@ class TransactionQueue(object): @defer.inlineCallbacks def enqueue_failure(self, failure, destination): - if destination == self.server_name: + if destination == self.server_name or destination == "localhost": return deferred = defer.Deferred() -- cgit 1.4.1 From 2462aacd77963431507bb97769acee3ed9d65ceb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Feb 2015 11:51:00 +0000 Subject: Restrict the destinations that synapse can talk to --- synapse/federation/transaction_queue.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index ae04774c76..4b5460c797 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -66,6 +66,26 @@ class TransactionQueue(object): # HACK to get unique tx id self._next_txn_id = int(self._clock.time_msec()) + def can_send_to(self, destination): + """Can we send messages to the given server? + + We can't send messages to ourselves. If we are running on localhost + then we can only federation with other servers running on localhost. + Otherwise we only federate with servers on a public domain. + + Args: + destination(str): The server we are possibly trying to send to. + Returns: + bool: True if we can send to the server. + """ + + if destination == self.server_name: + return False + if self.server_name.startswith("localhost"): + return destination.startswith("localhost") + else: + return not destination.startswith("localhost") + @defer.inlineCallbacks @log_function def enqueue_pdu(self, pdu, destinations, order): @@ -74,8 +94,9 @@ class TransactionQueue(object): # table and we'll get back to it later. destinations = set(destinations) - destinations.discard(self.server_name) - destinations.discard("localhost") + destinations = set( + dest for dest in destinations if self.can_send_to(dest) + ) logger.debug("Sending to: %s", str(destinations)) @@ -107,7 +128,7 @@ class TransactionQueue(object): def enqueue_edu(self, edu): destination = edu.destination - if destination == self.server_name: + if not self.can_send_to(destination): return deferred = defer.Deferred() @@ -130,6 +151,9 @@ class TransactionQueue(object): def enqueue_failure(self, failure, destination): deferred = defer.Deferred() + if not self.can_send_to(destination): + return + self.pending_failures_by_dest.setdefault( destination, [] ).append( -- cgit 1.4.1 From 6375bd3e33f687499e6d15298e74416c77ab499e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Feb 2015 12:01:37 +0000 Subject: SYN-282: Don't log tracebacks for client errors --- synapse/rest/media/v1/base_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index d44d5f1298..b10cbddb81 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -54,7 +54,7 @@ class BaseMediaResource(Resource): try: yield request_handler(self, request) except CodeMessageException as e: - logger.exception(e) + logger.info("Responding with error: %r", e) respond_with_json( request, e.code, cs_exception(e), send_cors=True ) -- cgit 1.4.1 From cc3d3babb0c04b21501f59c25739b9e48fa5b490 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 12:01:41 +0000 Subject: Remove unused import --- synapse/util/retryutils.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse') diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index ab58eb528b..4e82232796 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -17,8 +17,6 @@ from twisted.internet import defer from synapse.api.errors import CodeMessageException -from synapse.util.async import run_on_reactor - import logging -- cgit 1.4.1 From 446ef589920a9c3c4738873023494bed79d1f6c3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 12:03:12 +0000 Subject: Add errback to all deferreds in transaction_queue --- synapse/federation/transaction_queue.py | 37 ++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 14 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 06944eefb9..1e5505a4bf 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -90,14 +90,17 @@ class TransactionQueue(object): (pdu, deferred, order) ) - def eb(failure): + def chain(failure): if not deferred.called: deferred.errback(failure) - else: - logger.warn("Failed to send pdu", failure.value) + + def log_failure(failure): + logger.warn("Failed to send pdu", failure.value) + + deferred.addErrback(log_failure) with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) + self._attempt_new_transaction(destination).addErrback(chain) deferreds.append(deferred) @@ -115,14 +118,17 @@ class TransactionQueue(object): (edu, deferred) ) - def eb(failure): + def chain(failure): if not deferred.called: deferred.errback(failure) - else: - logger.warn("Failed to send edu", failure.value) + + def log_failure(failure): + logger.warn("Failed to send pdu", failure.value) + + deferred.addErrback(log_failure) with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) + self._attempt_new_transaction(destination).addErrback(chain) return deferred @@ -139,14 +145,17 @@ class TransactionQueue(object): (failure, deferred) ) - def eb(failure): + def chain(f): if not deferred.called: - deferred.errback(failure) - else: - logger.warn("Failed to send failure", failure.value) + deferred.errback(f) + + def log_failure(f): + logger.warn("Failed to send pdu", f.value) + + deferred.addErrback(log_failure) with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) + self._attempt_new_transaction(destination).addErrback(chain) yield deferred @@ -308,7 +317,7 @@ class TransactionQueue(object): except Exception as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. - logger.exception( + logger.warn( "TX [%s] Problem in _attempt_transaction: %s", destination, e, -- cgit 1.4.1 From d122e215ff459648d0352627c91adb00c6a34213 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 16:21:35 +0000 Subject: Generate a version string that includes git details if run from git checkout --- synapse/app/homeserver.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 351fc69143..166a0169ab 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -52,6 +52,7 @@ import synapse import logging import os import re +import subprocess import sqlite3 import syweb @@ -208,6 +209,61 @@ class SynapseHomeServer(HomeServer): logger.info("Synapse now listening on port %d", unsecure_port) +def get_version_string(): + null = open(os.devnull, 'w') + try: + git_branch = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], + stderr=null, + ).strip() + git_branch = "b=" + git_branch + except subprocess.CalledProcessError: + git_branch = "" + + try: + git_tag = subprocess.check_output( + ['git', 'describe', '--exact-match'], + stderr=null, + ).strip() + git_tag = "t=" + git_tag + except subprocess.CalledProcessError: + git_tag = "" + + try: + git_commit = subprocess.check_output( + ['git', 'rev-parse', '--short', 'HEAD'], + stderr=null, + ).strip() + except subprocess.CalledProcessError: + git_commit = "" + + try: + dirty_string = "-this_is_a_dirty_checkout" + is_dirty = subprocess.check_output( + ['git', 'describe', '--dirty=' + dirty_string], + stderr=null, + ).strip().endswith(dirty_string) + + git_dirty = "dirty" if is_dirty else "" + except subprocess.CalledProcessError: + git_dirty = "" + + if git_branch or git_tag or git_commit or git_dirty: + git_version = ",".join( + s for s in + (git_branch, git_tag, git_commit, git_dirty,) + if s + ) + + return ( + "Synapse/v=%s,%s" % ( + synapse.__version__, git_version, + ) + ).encode("ascii") + + return ("Synapse/%s" % (synapse.__version__,)).encode("ascii") + + def setup(): config = HomeServerConfig.load_config( "Synapse Homeserver", @@ -219,8 +275,10 @@ def setup(): check_requirements() + version_string = get_version_string() + logger.info("Server hostname: %s", config.server_name) - logger.info("Server version: %s", synapse.__version__) + logger.info("Server version: %s", version_string) if re.search(":[0-9]+$", config.server_name): domain_with_port = config.server_name @@ -229,6 +287,7 @@ def setup(): tls_context_factory = context_factory.ServerContextFactory(config) + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, @@ -237,6 +296,7 @@ def setup(): tls_context_factory=tls_context_factory, config=config, content_addr=config.content_addr, + version_string=version_string, ) hs.create_resource_tree( -- cgit 1.4.1 From aa577df06405c1574084389bcd189baedc3a7397 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 16:26:30 +0000 Subject: When computing git version run git commands in same dir as source files --- synapse/app/homeserver.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 166a0169ab..6acb3b2f4e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -211,10 +211,12 @@ class SynapseHomeServer(HomeServer): def get_version_string(): null = open(os.devnull, 'w') + cwd = os.path.dirname(os.path.abspath(__file__)) try: git_branch = subprocess.check_output( ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stderr=null, + cwd=cwd, ).strip() git_branch = "b=" + git_branch except subprocess.CalledProcessError: @@ -224,6 +226,7 @@ def get_version_string(): git_tag = subprocess.check_output( ['git', 'describe', '--exact-match'], stderr=null, + cwd=cwd, ).strip() git_tag = "t=" + git_tag except subprocess.CalledProcessError: @@ -233,6 +236,7 @@ def get_version_string(): git_commit = subprocess.check_output( ['git', 'rev-parse', '--short', 'HEAD'], stderr=null, + cwd=cwd, ).strip() except subprocess.CalledProcessError: git_commit = "" @@ -242,6 +246,7 @@ def get_version_string(): is_dirty = subprocess.check_output( ['git', 'describe', '--dirty=' + dirty_string], stderr=null, + cwd=cwd, ).strip().endswith(dirty_string) git_dirty = "dirty" if is_dirty else "" -- cgit 1.4.1 From 5358966a878aa543659c10be09f4bf58b1568f2f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 16:51:33 +0000 Subject: Use git aware version string in User-Agent and Server headers --- synapse/http/agent_name.py | 18 ------------------ synapse/http/client.py | 10 +++++----- synapse/http/matrixfederationclient.py | 4 ++-- synapse/http/server.py | 27 ++++++++++++++++++--------- tests/utils.py | 10 ++++++++-- 5 files changed, 33 insertions(+), 36 deletions(-) delete mode 100644 synapse/http/agent_name.py (limited to 'synapse') diff --git a/synapse/http/agent_name.py b/synapse/http/agent_name.py deleted file mode 100644 index d761890863..0000000000 --- a/synapse/http/agent_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse import __version__ - -AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii") diff --git a/synapse/http/client.py b/synapse/http/client.py index e46e7db146..22b7145ac1 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -14,7 +14,6 @@ # limitations under the License. from synapse.api.errors import CodeMessageException -from synapse.http.agent_name import AGENT_NAME from syutil.jsonutil import encode_canonical_json from twisted.internet import defer, reactor @@ -44,6 +43,7 @@ class SimpleHttpClient(object): # BrowserLikePolicyForHTTPS which will do regular cert validation # 'like a browser' self.agent = Agent(reactor) + self.version_string = hs.version_string @defer.inlineCallbacks def post_urlencoded_get_json(self, uri, args={}): @@ -55,7 +55,7 @@ class SimpleHttpClient(object): uri.encode("ascii"), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -108,7 +108,7 @@ class SimpleHttpClient(object): "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }) ) @@ -149,7 +149,7 @@ class SimpleHttpClient(object): "PUT", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], "Content-Type": ["application/json"] }), bodyProducer=FileBodyProducer(StringIO(json_str)) @@ -182,7 +182,7 @@ class CaptchaServerHttpClient(SimpleHttpClient): bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }) ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 49a4c7d9d3..9d483032a0 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -20,7 +20,6 @@ from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone -from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext @@ -80,6 +79,7 @@ class MatrixFederationHttpClient(object): self.server_name = hs.hostname self.agent = MatrixFederationHttpAgent(reactor) self.clock = hs.get_clock() + self.version_string = hs.version_string @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, @@ -87,7 +87,7 @@ class MatrixFederationHttpClient(object): query_bytes=b"", retry_on_dns_fail=True): """ Creates and sends a request to the given url """ - headers_dict[b"User-Agent"] = [AGENT_NAME] + headers_dict[b"User-Agent"] = [self.version_string] headers_dict[b"Host"] = [destination] url_bytes = urlparse.urlunparse( diff --git a/synapse/http/server.py b/synapse/http/server.py index 589c30c199..74a101a5d7 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -14,7 +14,6 @@ # limitations under the License. -from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError ) @@ -74,6 +73,7 @@ class JsonResource(HttpServer, resource.Resource): self.clock = hs.get_clock() self.path_regexs = {} + self.version_string = hs.version_string def register_path(self, method, path_pattern, callback): self.path_regexs.setdefault(method, []).append( @@ -189,9 +189,13 @@ class JsonResource(HttpServer, resource.Resource): return # TODO: Only enable CORS for the requests that need it. - respond_with_json(request, code, response_json_object, send_cors=True, - response_code_message=response_code_message, - pretty_print=self._request_user_agent_is_curl) + respond_with_json( + request, code, response_json_object, + send_cors=True, + response_code_message=response_code_message, + pretty_print=self._request_user_agent_is_curl, + version_string=self.version_string, + ) @staticmethod def _request_user_agent_is_curl(request): @@ -221,18 +225,23 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, - response_code_message=None, pretty_print=False): + response_code_message=None, pretty_print=False, + version_string=""): if not pretty_print: json_bytes = encode_pretty_printed_json(json_object) else: json_bytes = encode_canonical_json(json_object) - return respond_with_json_bytes(request, code, json_bytes, send_cors, - response_code_message=response_code_message) + return respond_with_json_bytes( + request, code, json_bytes, + send_cors=send_cors, + response_code_message=response_code_message, + version_string=version_string + ) def respond_with_json_bytes(request, code, json_bytes, send_cors=False, - response_code_message=None): + version_string="", response_code_message=None): """Sends encoded JSON in response to the given request. Args: @@ -246,7 +255,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") - request.setHeader(b"Server", AGENT_NAME) + request.setHeader(b"Server", version_string) request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) if send_cors: diff --git a/tests/utils.py b/tests/utils.py index 39895c739f..110b9f86b8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -46,10 +46,16 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): if datastore is None: db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() - hs = HomeServer(name, db_pool=db_pool, config=config, **kargs) + hs = HomeServer( + name, db_pool=db_pool, config=config, + version_string="Synapse/tests", + **kargs + ) else: hs = HomeServer( - name, db_pool=None, datastore=datastore, config=config, **kargs + name, db_pool=None, datastore=datastore, config=config, + version_string="Synapse/tests", + **kargs ) defer.returnValue(hs) -- cgit 1.4.1 From 1df8bad63e7f55dca79f20919a6873157c76d8b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 16:54:25 +0000 Subject: pyflakes --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6acb3b2f4e..3368ec6759 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -292,7 +292,6 @@ def setup(): tls_context_factory = context_factory.ServerContextFactory(config) - hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, -- cgit 1.4.1 From 5b753d472bd4ffba124103753f83b55c567889fb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 17:02:40 +0000 Subject: Bump matrix-angular-sdk version --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 9464b4fb62..c71df75404 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil>=0.0.3": ["syutil"], - "matrix_angular_sdk>=0.6.2": ["syweb>=0.6.2"], + "matrix_angular_sdk>=0.6.3": ["syweb>=0.6.3"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -36,8 +36,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/matrix-angular-sdk", - version="v0.6.2", - egg="matrix_angular_sdk-0.6.2", + version="v0.6.3", + egg="matrix_angular_sdk-0.6.3", ), ] -- cgit 1.4.1 From b6771037a6d726803f76d486bf82075553d4a7b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 17:31:50 +0000 Subject: Make version_string conform to User-Agent and Server spec --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 3368ec6759..bf01483b85 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -261,7 +261,7 @@ def get_version_string(): ) return ( - "Synapse/v=%s,%s" % ( + "Synapse/%s/(%s)" % ( synapse.__version__, git_version, ) ).encode("ascii") -- cgit 1.4.1 From 3f1871021e4efd1bbc513790e5c82f09b4c04503 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 17:32:12 +0000 Subject: Make /keys/ return correct Server version --- synapse/http/server_key_resource.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/http/server_key_resource.py b/synapse/http/server_key_resource.py index 4fc491dc82..71e9a51f5c 100644 --- a/synapse/http/server_key_resource.py +++ b/synapse/http/server_key_resource.py @@ -50,6 +50,7 @@ class LocalKey(Resource): def __init__(self, hs): self.hs = hs + self.version_string = hs.version_string self.response_body = encode_canonical_json( self.response_json_object(hs.config) ) @@ -82,7 +83,10 @@ class LocalKey(Resource): return json_object def render_GET(self, request): - return respond_with_json_bytes(request, 200, self.response_body) + return respond_with_json_bytes( + request, 200, self.response_body, + version_string=self.version_string + ) def getChild(self, name, request): if name == '': -- cgit 1.4.1 From 6311ae8968d5f2e3b867789376933de83678e036 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 17:34:26 +0000 Subject: Conform to header spec take two --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bf01483b85..ea20de1434 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -261,7 +261,7 @@ def get_version_string(): ) return ( - "Synapse/%s/(%s)" % ( + "Synapse/%s (%s)" % ( synapse.__version__, git_version, ) ).encode("ascii") -- cgit 1.4.1 From b457f1677c0798894140eb6b7e091800c7c7bf2d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 19 Feb 2015 10:06:17 +0000 Subject: Send room ID in http notifications so clients know which room to go to if the user responds to the notification. --- synapse/push/httppusher.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 5788db4eba..202fcb42f4 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -66,6 +66,7 @@ class HttpPusher(Pusher): d = { 'notification': { 'id': event['event_id'], + 'room_id': event['room_id'], 'type': event['type'], 'sender': event['user_id'], 'counts': { # -- we don't mark messages as read yet so -- cgit 1.4.1 From c3eb7dd9c572cfcea831e4d5f41b5a34cb3c57e8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 11:50:49 +0000 Subject: Add config option to set the soft fd limit on start --- synapse/app/homeserver.py | 20 +++++++++++++++++--- synapse/config/server.py | 7 +++++++ 2 files changed, 24 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ea20de1434..07386f6f28 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -52,6 +52,7 @@ import synapse import logging import os import re +import resource import subprocess import sqlite3 import syweb @@ -269,6 +270,15 @@ def get_version_string(): return ("Synapse/%s" % (synapse.__version__,)).encode("ascii") +def change_resource_limit(soft_file_no): + try: + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) + logger.info("Set file limit to: %d", soft_file_no) + except (ValueError, resource.error) as e: + logger.warn("Failed to set file limit: %s", e) + + def setup(): config = HomeServerConfig.load_config( "Synapse Homeserver", @@ -345,10 +355,11 @@ def setup(): if config.daemonize: print config.pid_file + daemon = Daemonize( app="synapse-homeserver", pid=config.pid_file, - action=run, + action=lambda: run(config), auto_close_fds=False, verbose=True, logger=logger, @@ -356,11 +367,14 @@ def setup(): daemon.start() else: - reactor.run() + run(config) -def run(): +def run(config): with LoggingContext("run"): + if config.soft_file_limit: + change_resource_limit(config.soft_file_limit) + reactor.run() diff --git a/synapse/config/server.py b/synapse/config/server.py index 31e44cc857..a3edf208e9 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -31,6 +31,7 @@ class ServerConfig(Config): self.webclient = True self.manhole = args.manhole self.no_tls = args.no_tls + self.soft_file_limit = args.soft_file_limit if not args.content_addr: host = args.server_name @@ -77,6 +78,12 @@ class ServerConfig(Config): "content repository") server_group.add_argument("--no-tls", action='store_true', help="Don't bind to the https port.") + server_group.add_argument("--soft-file-limit", type=int, default=0, + help="Set the limit on the number of file " + "descriptors synapse can use. Zero " + "is used to indicate synapse should " + "not change the limit from system " + "default.") def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") -- cgit 1.4.1 From 939273c4b0936b95eb14763d8bf1027159fa4cb3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 11:53:13 +0000 Subject: Rename resource variable so as to not shadow module import --- synapse/app/homeserver.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 07386f6f28..5f671dfd23 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -141,8 +141,8 @@ class SynapseHomeServer(HomeServer): # instead, we'll store a copy of this mapping so we can actually add # extra resources to existing nodes. See self._resource_id for the key. resource_mappings = {} - for (full_path, resource) in desired_tree: - logger.info("Attaching %s to path %s", resource, full_path) + for full_path, res in desired_tree: + logger.info("Attaching %s to path %s", res, full_path) last_resource = self.root_resource for path_seg in full_path.split('/')[1:-1]: if path_seg not in last_resource.listNames(): @@ -173,12 +173,12 @@ class SynapseHomeServer(HomeServer): child_name) child_resource = resource_mappings[child_res_id] # steal the children - resource.putChild(child_name, child_resource) + res.putChild(child_name, child_resource) # finally, insert the desired resource in the right place - last_resource.putChild(last_path_seg, resource) + last_resource.putChild(last_path_seg, res) res_id = self._resource_id(last_resource, last_path_seg) - resource_mappings[res_id] = resource + resource_mappings[res_id] = res return self.root_resource -- cgit 1.4.1 From 81163f822e5133ae6e724f18e70e9b84b2081a4e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 14:16:53 +0000 Subject: Add config option to disable registration. --- synapse/config/homeserver.py | 3 ++- synapse/config/registration.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 synapse/config/registration.py (limited to 'synapse') diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index b0fe217459..c024535f52 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -22,11 +22,12 @@ from .repository import ContentRepositoryConfig from .captcha import CaptchaConfig from .email import EmailConfig from .voip import VoipConfig +from .registration import RegistrationConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, - EmailConfig, VoipConfig): + EmailConfig, VoipConfig, RegistrationConfig,): pass diff --git a/synapse/config/registration.py b/synapse/config/registration.py new file mode 100644 index 0000000000..cca8ab5676 --- /dev/null +++ b/synapse/config/registration.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class RegistrationConfig(Config): + + def __init__(self, args): + super(RegistrationConfig, self).__init__(args) + self.disable_registration = args.disable_registration + + @classmethod + def add_arguments(cls, parser): + super(RegistrationConfig, cls).add_arguments(parser) + reg_group = parser.add_argument_group("registration") + reg_group.add_argument( + "--disable-registration", + action='store_true', + help="Disable registration of new users." + ) -- cgit 1.4.1 From 0722f982d3b664e6e1c20c3853bb3742683aee88 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 14:22:20 +0000 Subject: Disable registration if config option was set. --- synapse/rest/client/v1/register.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'synapse') diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index 8d2115082b..dba49129a9 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -59,6 +59,7 @@ class RegisterRestServlet(ClientV1RestServlet): # } # TODO: persistent storage self.sessions = {} + self.disable_registration = hs.config.disable_registration def on_GET(self, request): if self.hs.config.enable_registration_captcha: @@ -97,6 +98,9 @@ class RegisterRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): + if self.disable_registration: + raise SynapseError(403, "Registration has been disabled") + register_json = _parse_json(request) session = (register_json["session"] -- cgit 1.4.1 From 61959928bb4b6b0191d301f1b267af7290a61bd2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 19 Feb 2015 14:58:07 +0000 Subject: Pull out the 'get_rooms_for_user' cache logic into a reüsable @cached decorator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- synapse/storage/roommember.py | 53 +++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 24 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 9bf608bc90..569bd55d0f 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -33,6 +33,32 @@ RoomsForUser = namedtuple( ) +# TODO(paul): +# * Move this somewhere higher-level, shared; +# * more generic key management +# * export monitoring stats +# * maximum size; just evict things at random, or consider LRU? +def cached(orig): + cache = {} + + @defer.inlineCallbacks + def wrapped(self, key): + if key in cache: + defer.returnValue(cache[key]) + + ret = yield orig(self, key) + + cache[key] = ret; + defer.returnValue(ret) + + def invalidate(key): + if key in cache: + del cache[key] + + wrapped.invalidate = invalidate + return wrapped + + class RoomMemberStore(SQLBaseStore): def __init__(self, *args, **kw): @@ -103,7 +129,7 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, (event.room_id, domain)) - self.invalidate_rooms_for_user(target_user_id) + self.get_rooms_for_user.invalidate(target_user_id) @defer.inlineCallbacks def get_room_member(self, user_id, room_id): @@ -247,33 +273,12 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results - # TODO(paul): Create a nice @cached decorator to do this - # @cached - # def get_foo(...) - # ... - # invalidate_foo = get_foo.invalidator - - @defer.inlineCallbacks + @cached def get_rooms_for_user(self, user_id): - # TODO(paul): put some performance counters in here so we can easily - # track what impact this cache is having - if user_id in self._user_rooms_cache: - defer.returnValue(self._user_rooms_cache[user_id]) - - rooms = yield self.get_rooms_for_user_where_membership_is( + return self.get_rooms_for_user_where_membership_is( user_id, membership_list=[Membership.JOIN], ) - # TODO(paul): Consider applying a maximum size; just evict things at - # random, or consider LRU? - - self._user_rooms_cache[user_id] = rooms - defer.returnValue(rooms) - - def invalidate_rooms_for_user(self, user_id): - if user_id in self._user_rooms_cache: - del self._user_rooms_cache[user_id] - @defer.inlineCallbacks def user_rooms_intersect(self, user_id_list): """ Checks whether all the users whose IDs are given in a list share a -- cgit 1.4.1 From 0ac2a79faa918280767c18e4db7ec29d7d3a3afb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 17:24:14 +0000 Subject: Initial stab at implementing a batched get_missing_pdus request --- synapse/federation/federation_server.py | 72 +++++++++++++++++++++++++++++++++ synapse/handlers/federation.py | 9 +++-- synapse/storage/event_federation.py | 63 ++++++++++++++++++++++++++--- 3 files changed, 135 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 22b9663831..34bc397e8a 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -305,6 +305,78 @@ class FederationServer(FederationBase): (200, send_content) ) + @defer.inlineCallbacks + def get_missing_events(self, origin, room_id, earliest_events, + latest_events, limit, min_depth): + limit = max(limit, 50) + min_depth = max(min_depth, 0) + + missing_events = yield self.store.get_missing_events( + room_id=room_id, + earliest_events=earliest_events, + latest_events=latest_events, + limit=limit, + min_depth=min_depth, + ) + + known_ids = {e.event_id for e in missing_events} | {earliest_events} + + back_edges = { + e for e in missing_events + if {i for i, h in e.prev_events.items()} <= known_ids + } + + decoded_auth_events = set() + state = {} + auth_events = set() + auth_and_state = {} + for event in back_edges: + state_pdus = yield self.handler.get_state_for_pdu( + origin, room_id, event.event_id, + do_auth=False, + ) + + state[event.event_id] = [s.event_id for s in state_pdus] + + auth_and_state.update({ + s.event_id: s for s in state_pdus + }) + + state_ids = {pdu.event_id for pdu in state_pdus} + prev_ids = {i for i, h in event.prev_events.items()} + partial_auth_chain = yield self.store.get_auth_chain( + state_ids | prev_ids, have_ids=decoded_auth_events.keys() + ) + + for p in partial_auth_chain: + p.signatures.update( + compute_event_signature( + p, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) + + auth_events.update( + a.event_id for a in partial_auth_chain + ) + + auth_and_state.update({ + a.event_id: a for a in partial_auth_chain + }) + + time_now = self._clock.time_msec() + + defer.returnValue({ + "events": [ev.get_pdu_json(time_now) for ev in missing_events], + "state_for_events": state, + "auth_events": auth_events, + "event_map": { + k: ev.get_pdu_json(time_now) + for k, ev in auth_and_state.items() + }, + }) + @log_function def _get_persisted_pdu(self, origin, event_id, do_auth=True): """ Get a PDU from the database with given origin and id. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0eb2ff95ca..26bdc6d1a7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -581,12 +581,13 @@ class FederationHandler(BaseHandler): defer.returnValue(event) @defer.inlineCallbacks - def get_state_for_pdu(self, origin, room_id, event_id): + def get_state_for_pdu(self, origin, room_id, event_id, do_auth=True): yield run_on_reactor() - in_room = yield self.auth.check_host_in_room(room_id, origin) - if not in_room: - raise AuthError(403, "Host not in room.") + if do_auth: + in_room = yield self.auth.check_host_in_room(room_id, origin) + if not in_room: + raise AuthError(403, "Host not in room.") state_groups = yield self.store.get_state_groups( [event_id] diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 3fbc090224..22bf7ad832 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -32,15 +32,15 @@ class EventFederationStore(SQLBaseStore): and backfilling from another server respectively. """ - def get_auth_chain(self, event_ids): + def get_auth_chain(self, event_ids, have_ids=set()): return self.runInteraction( "get_auth_chain", self._get_auth_chain_txn, - event_ids + event_ids, have_ids ) - def _get_auth_chain_txn(self, txn, event_ids): - results = self._get_auth_chain_ids_txn(txn, event_ids) + def _get_auth_chain_txn(self, txn, event_ids, have_ids): + results = self._get_auth_chain_ids_txn(txn, event_ids, have_ids) return self._get_events_txn(txn, results) @@ -51,8 +51,9 @@ class EventFederationStore(SQLBaseStore): event_ids ) - def _get_auth_chain_ids_txn(self, txn, event_ids): + def _get_auth_chain_ids_txn(self, txn, event_ids, have_ids): results = set() + have_ids = set(have_ids) base_sql = ( "SELECT auth_id FROM event_auth WHERE event_id = ?" @@ -64,6 +65,10 @@ class EventFederationStore(SQLBaseStore): for f in front: txn.execute(base_sql, (f,)) new_front.update([r[0] for r in txn.fetchall()]) + + new_front -= results + new_front -= have_ids + front = new_front results.update(front) @@ -378,3 +383,51 @@ class EventFederationStore(SQLBaseStore): event_results += new_front return self._get_events_txn(txn, event_results) + + def get_missing_events(self, room_id, earliest_events, latest_events, + limit, min_depth): + return self.runInteraction( + "get_missing_events", + self._get_missing_events, + room_id, earliest_events, latest_events, limit, min_depth + ) + + def _get_missing_events(self, txn, room_id, earliest_events, latest_events, + limit, min_depth): + + earliest_events = set(earliest_events) + front = set(latest_events) - earliest_events + + event_results = set() + + query = ( + "SELECT prev_event_id FROM event_edges " + "WHERE room_id = ? AND event_id = ? AND is_state = 0 " + "LIMIT ?" + ) + + while front and len(event_results) < limit: + new_front = set() + for event_id in front: + txn.execute( + query, + (room_id, event_id, limit - len(event_results)) + ) + + for e_id, in txn.fetchall(): + new_front.add(e_id) + + new_front -= earliest_events + new_front -= event_results + + front = new_front + event_results |= new_front + + events = self._get_events_txn(txn, event_results) + + events = sorted( + [ev for ev in events if ev.depth >= min_depth], + key=lambda e: e.depth, + ) + + return events[:limit] -- cgit 1.4.1 From 077d20034278ea57c57d501de11bfb1f0c7f9603 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 19 Feb 2015 17:29:39 +0000 Subject: Move @cached decorator out into synapse.storage._base; add minimal docs --- synapse/storage/_base.py | 35 +++++++++++++++++++++++++++++++++++ synapse/storage/roommember.py | 28 +--------------------------- 2 files changed, 36 insertions(+), 27 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index be9934c66f..fd275039be 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -35,6 +35,41 @@ sql_logger = logging.getLogger("synapse.storage.SQL") transaction_logger = logging.getLogger("synapse.storage.txn") +# TODO(paul): +# * Move this somewhere higher-level, shared; +# * more generic key management +# * export monitoring stats +# * maximum size; just evict things at random, or consider LRU? +def cached(orig): + """ A method decorator that applies a memoizing cache around the function. + + The function is presumed to take one additional argument, which is used as + the key for the cache. Cache hits are served directly from the cache; + misses use the function body to generate the value. + + The wrapped function has an additional member, a callable called + "invalidate". This can be used to remove individual entries from the cache. + """ + cache = {} + + @defer.inlineCallbacks + def wrapped(self, key): + if key in cache: + defer.returnValue(cache[key]) + + ret = yield orig(self, key) + + cache[key] = ret; + defer.returnValue(ret) + + def invalidate(key): + if key in cache: + del cache[key] + + wrapped.invalidate = invalidate + return wrapped + + class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging to the .execute() method.""" diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 569bd55d0f..b8fcc1927e 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -17,7 +17,7 @@ from twisted.internet import defer from collections import namedtuple -from ._base import SQLBaseStore +from ._base import SQLBaseStore, cached from synapse.api.constants import Membership from synapse.types import UserID @@ -33,32 +33,6 @@ RoomsForUser = namedtuple( ) -# TODO(paul): -# * Move this somewhere higher-level, shared; -# * more generic key management -# * export monitoring stats -# * maximum size; just evict things at random, or consider LRU? -def cached(orig): - cache = {} - - @defer.inlineCallbacks - def wrapped(self, key): - if key in cache: - defer.returnValue(cache[key]) - - ret = yield orig(self, key) - - cache[key] = ret; - defer.returnValue(ret) - - def invalidate(key): - if key in cache: - del cache[key] - - wrapped.invalidate = invalidate - return wrapped - - class RoomMemberStore(SQLBaseStore): def __init__(self, *args, **kw): -- cgit 1.4.1 From ebc3db295bfe6d0c43bf45b8fcd7fa6bbc429375 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 19 Feb 2015 18:36:02 +0000 Subject: Take named arguments to @cached() decorator, add a 'max_entries' limit --- synapse/storage/_base.py | 39 +++++++++++-------- synapse/storage/roommember.py | 2 +- tests/storage/test__base.py | 89 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 16 deletions(-) create mode 100644 tests/storage/test__base.py (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index fd275039be..61657d36ed 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -39,8 +39,8 @@ transaction_logger = logging.getLogger("synapse.storage.txn") # * Move this somewhere higher-level, shared; # * more generic key management # * export monitoring stats -# * maximum size; just evict things at random, or consider LRU? -def cached(orig): +# * consider other eviction strategies - LRU? +def cached(max_entries=1000): """ A method decorator that applies a memoizing cache around the function. The function is presumed to take one additional argument, which is used as @@ -50,24 +50,33 @@ def cached(orig): The wrapped function has an additional member, a callable called "invalidate". This can be used to remove individual entries from the cache. """ - cache = {} + def wrap(orig): + cache = {} - @defer.inlineCallbacks - def wrapped(self, key): - if key in cache: - defer.returnValue(cache[key]) + @defer.inlineCallbacks + def wrapped(self, key): + if key in cache: + defer.returnValue(cache[key]) + + ret = yield orig(self, key) + + while len(cache) > max_entries: + # TODO(paul): This feels too biased. However, a random index + # would be a bit inefficient, walking the list of keys just + # to ignore most of them? + del cache[cache.keys()[0]] - ret = yield orig(self, key) + cache[key] = ret; + defer.returnValue(ret) - cache[key] = ret; - defer.returnValue(ret) + def invalidate(key): + if key in cache: + del cache[key] - def invalidate(key): - if key in cache: - del cache[key] + wrapped.invalidate = invalidate + return wrapped - wrapped.invalidate = invalidate - return wrapped + return wrap class LoggingTransaction(object): diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index b8fcc1927e..33a832483e 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -247,7 +247,7 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results - @cached + @cached() def get_rooms_for_user(self, user_id): return self.get_rooms_for_user_where_membership_is( user_id, membership_list=[Membership.JOIN], diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py new file mode 100644 index 0000000000..057f798640 --- /dev/null +++ b/tests/storage/test__base.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest +from twisted.internet import defer + +from synapse.storage._base import cached + + +class CacheDecoratorTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def test_passthrough(self): + @cached() + def func(self, key): + return key + + self.assertEquals((yield func(self, "foo")), "foo") + self.assertEquals((yield func(self, "bar")), "bar") + + @defer.inlineCallbacks + def test_hit(self): + callcount = [0] + + @cached() + def func(self, key): + callcount[0] += 1 + return key + + yield func(self, "foo") + + self.assertEquals(callcount[0], 1) + + self.assertEquals((yield func(self, "foo")), "foo") + self.assertEquals(callcount[0], 1) + + @defer.inlineCallbacks + def test_invalidate(self): + callcount = [0] + + @cached() + def func(self, key): + callcount[0] += 1 + return key + + yield func(self, "foo") + + self.assertEquals(callcount[0], 1) + + func.invalidate("foo") + + yield func(self, "foo") + + self.assertEquals(callcount[0], 2) + + @defer.inlineCallbacks + def test_max_entries(self): + callcount = [0] + + @cached(max_entries=10) + def func(self, key): + callcount[0] += 1 + return key + + for k in range(0,12): + yield func(self, k) + + self.assertEquals(callcount[0], 12) + + # There must have been at least 2 evictions, meaning if we calculate + # all 12 values again, we must get called at least 2 more times + for k in range(0,12): + yield func(self, k) + + self.assertTrue(callcount[0] >= 14, + msg="Expected callcount >= 14, got %d" % (callcount[0])) -- cgit 1.4.1 From 55022d6ca5bfe3e99fd3144e291906063885ce12 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 19 Feb 2015 18:38:09 +0000 Subject: Remove a TODO note --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 61657d36ed..78ba5f25ea 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -36,7 +36,6 @@ transaction_logger = logging.getLogger("synapse.storage.txn") # TODO(paul): -# * Move this somewhere higher-level, shared; # * more generic key management # * export monitoring stats # * consider other eviction strategies - LRU? -- cgit 1.4.1 From 15e2d7e38762efbc9e9c706d4d2a63bb6fcfd541 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 11:39:53 +0000 Subject: Always allow AS to register --- synapse/rest/client/v1/register.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index dba49129a9..f5acfb945f 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -98,9 +98,6 @@ class RegisterRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - if self.disable_registration: - raise SynapseError(403, "Registration has been disabled") - register_json = _parse_json(request) session = (register_json["session"] @@ -111,6 +108,11 @@ class RegisterRestServlet(ClientV1RestServlet): try: login_type = register_json["type"] + + is_application_server = login_type == LoginType.APPLICATION_SERVICE + if self.disable_registration and not is_application_server: + raise SynapseError(403, "Registration has been disabled") + stages = { LoginType.RECAPTCHA: self._do_recaptcha, LoginType.PASSWORD: self._do_password, -- cgit 1.4.1 From 2bafeca2703635ea1fee2a7617c30cfad1b00315 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 14:08:42 +0000 Subject: Add missing comma so that it generates a dict and not a set --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0eb2ff95ca..7deed16f9c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -875,7 +875,7 @@ class FederationHandler(BaseHandler): local_view = dict(auth_events) remote_view = dict(auth_events) remote_view.update({ - (d.type, d.state_key) for d in different_events + (d.type, d.state_key): d for d in different_events }) new_state, prev_state = self.state_handler.resolve_events( -- cgit 1.4.1 From 0edfecc9043feb294ef09a2d382d7c52d258a809 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 14:14:28 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index a804a25748..b39b01ac3c 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.1" +__version__ = "0.7.1.1" -- cgit 1.4.1 From dc0ee55110c1bcb0e3be39a07403e43763c0af25 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 15:00:14 +0000 Subject: Change version scheme --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index b39b01ac3c..f8ec4b0459 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.1.1" +__version__ = "0.7.1-r1" -- cgit 1.4.1 From 7c56210f204ef735444129c4f7b2a393eb7539ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 16:09:44 +0000 Subject: By default set soft limit to hard limit --- synapse/app/homeserver.py | 8 ++++++-- synapse/config/server.py | 10 +++++----- 2 files changed, 11 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5f671dfd23..89af091f10 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -273,7 +273,12 @@ def get_version_string(): def change_resource_limit(soft_file_no): try: soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + + if not soft_file_no: + soft_file_no = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) + logger.info("Set file limit to: %d", soft_file_no) except (ValueError, resource.error) as e: logger.warn("Failed to set file limit: %s", e) @@ -372,8 +377,7 @@ def setup(): def run(config): with LoggingContext("run"): - if config.soft_file_limit: - change_resource_limit(config.soft_file_limit) + change_resource_limit(config.soft_file_limit) reactor.run() diff --git a/synapse/config/server.py b/synapse/config/server.py index a3edf208e9..4e4892d40b 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -79,11 +79,11 @@ class ServerConfig(Config): server_group.add_argument("--no-tls", action='store_true', help="Don't bind to the https port.") server_group.add_argument("--soft-file-limit", type=int, default=0, - help="Set the limit on the number of file " - "descriptors synapse can use. Zero " - "is used to indicate synapse should " - "not change the limit from system " - "default.") + help="Set the soft limit on the number of " + "file descriptors synapse can use. " + "Zero is used to indicate synapse " + "should set the soft limit to the hard" + "limit.") def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") -- cgit 1.4.1 From 1f29fafc95af65c259c19c8ef90758328ac9e810 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 21 Feb 2015 13:44:46 +0000 Subject: Don't exit if we can't work out if we're running in a git repo --- synapse/app/homeserver.py | 109 ++++++++++++++++++++++++---------------------- 1 file changed, 56 insertions(+), 53 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ea20de1434..2c463936cb 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -210,61 +210,64 @@ class SynapseHomeServer(HomeServer): def get_version_string(): - null = open(os.devnull, 'w') - cwd = os.path.dirname(os.path.abspath(__file__)) try: - git_branch = subprocess.check_output( - ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], - stderr=null, - cwd=cwd, - ).strip() - git_branch = "b=" + git_branch - except subprocess.CalledProcessError: - git_branch = "" - - try: - git_tag = subprocess.check_output( - ['git', 'describe', '--exact-match'], - stderr=null, - cwd=cwd, - ).strip() - git_tag = "t=" + git_tag - except subprocess.CalledProcessError: - git_tag = "" - - try: - git_commit = subprocess.check_output( - ['git', 'rev-parse', '--short', 'HEAD'], - stderr=null, - cwd=cwd, - ).strip() - except subprocess.CalledProcessError: - git_commit = "" - - try: - dirty_string = "-this_is_a_dirty_checkout" - is_dirty = subprocess.check_output( - ['git', 'describe', '--dirty=' + dirty_string], - stderr=null, - cwd=cwd, - ).strip().endswith(dirty_string) - - git_dirty = "dirty" if is_dirty else "" - except subprocess.CalledProcessError: - git_dirty = "" - - if git_branch or git_tag or git_commit or git_dirty: - git_version = ",".join( - s for s in - (git_branch, git_tag, git_commit, git_dirty,) - if s - ) - - return ( - "Synapse/%s (%s)" % ( - synapse.__version__, git_version, + null = open(os.devnull, 'w') + cwd = os.path.dirname(os.path.abspath(__file__)) + try: + git_branch = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], + stderr=null, + cwd=cwd, + ).strip() + git_branch = "b=" + git_branch + except subprocess.CalledProcessError: + git_branch = "" + + try: + git_tag = subprocess.check_output( + ['git', 'describe', '--exact-match'], + stderr=null, + cwd=cwd, + ).strip() + git_tag = "t=" + git_tag + except subprocess.CalledProcessError: + git_tag = "" + + try: + git_commit = subprocess.check_output( + ['git', 'rev-parse', '--short', 'HEAD'], + stderr=null, + cwd=cwd, + ).strip() + except subprocess.CalledProcessError: + git_commit = "" + + try: + dirty_string = "-this_is_a_dirty_checkout" + is_dirty = subprocess.check_output( + ['git', 'describe', '--dirty=' + dirty_string], + stderr=null, + cwd=cwd, + ).strip().endswith(dirty_string) + + git_dirty = "dirty" if is_dirty else "" + except subprocess.CalledProcessError: + git_dirty = "" + + if git_branch or git_tag or git_commit or git_dirty: + git_version = ",".join( + s for s in + (git_branch, git_tag, git_commit, git_dirty,) + if s ) - ).encode("ascii") + + return ( + "Synapse/%s (%s)" % ( + synapse.__version__, git_version, + ) + ).encode("ascii") + except Exception as e: + logger.warn("Failed to check for git repository: %s", e) return ("Synapse/%s" % (synapse.__version__,)).encode("ascii") -- cgit 1.4.1 From a87c56c67327fcacc008c169a8f7fc244c6d8016 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 21 Feb 2015 13:45:07 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index f8ec4b0459..7d548b0a8a 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.1-r1" +__version__ = "0.7.1-r2" -- cgit 1.4.1 From db215b7e0007a207b8775d78c6693153e16f2731 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Feb 2015 13:58:02 +0000 Subject: Implement and use new batched get missing pdu --- synapse/federation/federation_client.py | 19 ++++ synapse/federation/federation_server.py | 150 +++++++++++--------------------- synapse/federation/transaction_queue.py | 2 +- synapse/federation/transport/client.py | 19 ++++ synapse/federation/transport/server.py | 31 +++++++ synapse/handlers/federation.py | 23 +++++ 6 files changed, 144 insertions(+), 100 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index cd3c962d50..ca89a0787c 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -439,6 +439,25 @@ class FederationClient(FederationBase): defer.returnValue(ret) + @defer.inlineCallbacks + def get_missing_events(self, destination, room_id, earliest_events, + latest_events, limit, min_depth): + content = yield self.transport_layer.get_missing_events( + destination, room_id, earliest_events, latest_events, limit, + min_depth, + ) + + events = [ + self.event_from_pdu_json(e) + for e in content.get("events", []) + ] + + signed_events = yield self._check_sigs_and_hash_and_fetch( + destination, events, outlier=True + ) + + defer.returnValue(signed_events) + def event_from_pdu_json(self, pdu_json, outlier=False): event = FrozenEvent( pdu_json diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 34bc397e8a..f74e16abd5 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -142,7 +142,15 @@ class FederationServer(FederationBase): if r[0]: ret.append({}) else: - logger.exception(r[1]) + failure = r[1] + logger.error( + "Failed to handle PDU", + exc_info=( + failure.type, + failure.value, + failure.getTracebackObject() + ) + ) ret.append({"error": str(r[1].value)}) logger.debug("Returning: %s", str(ret)) @@ -306,75 +314,17 @@ class FederationServer(FederationBase): ) @defer.inlineCallbacks - def get_missing_events(self, origin, room_id, earliest_events, - latest_events, limit, min_depth): - limit = max(limit, 50) - min_depth = max(min_depth, 0) - - missing_events = yield self.store.get_missing_events( - room_id=room_id, - earliest_events=earliest_events, - latest_events=latest_events, - limit=limit, - min_depth=min_depth, + @log_function + def on_get_missing_events(self, origin, room_id, earliest_events, + latest_events, limit, min_depth): + missing_events = yield self.handler.on_get_missing_events( + origin, room_id, earliest_events, latest_events, limit, min_depth ) - known_ids = {e.event_id for e in missing_events} | {earliest_events} - - back_edges = { - e for e in missing_events - if {i for i, h in e.prev_events.items()} <= known_ids - } - - decoded_auth_events = set() - state = {} - auth_events = set() - auth_and_state = {} - for event in back_edges: - state_pdus = yield self.handler.get_state_for_pdu( - origin, room_id, event.event_id, - do_auth=False, - ) - - state[event.event_id] = [s.event_id for s in state_pdus] - - auth_and_state.update({ - s.event_id: s for s in state_pdus - }) - - state_ids = {pdu.event_id for pdu in state_pdus} - prev_ids = {i for i, h in event.prev_events.items()} - partial_auth_chain = yield self.store.get_auth_chain( - state_ids | prev_ids, have_ids=decoded_auth_events.keys() - ) - - for p in partial_auth_chain: - p.signatures.update( - compute_event_signature( - p, - self.hs.hostname, - self.hs.config.signing_key[0] - ) - ) - - auth_events.update( - a.event_id for a in partial_auth_chain - ) - - auth_and_state.update({ - a.event_id: a for a in partial_auth_chain - }) - time_now = self._clock.time_msec() defer.returnValue({ "events": [ev.get_pdu_json(time_now) for ev in missing_events], - "state_for_events": state, - "auth_events": auth_events, - "event_map": { - k: ev.get_pdu_json(time_now) - for k, ev in auth_and_state.items() - }, }) @log_function @@ -403,7 +353,7 @@ class FederationServer(FederationBase): @defer.inlineCallbacks @log_function - def _handle_new_pdu(self, origin, pdu, max_recursion=10): + def _handle_new_pdu(self, origin, pdu, get_missing=True): # We reprocess pdus when we have seen them only as outliers existing = yield self._get_persisted_pdu( origin, pdu.event_id, do_auth=False @@ -455,48 +405,50 @@ class FederationServer(FederationBase): pdu.room_id, min_depth ) + prevs = {e_id for e_id, _ in pdu.prev_events} + seen = set(have_seen.keys()) + if min_depth and pdu.depth < min_depth: # This is so that we don't notify the user about this # message, to work around the fact that some events will # reference really really old events we really don't want to # send to the clients. pdu.internal_metadata.outlier = True - elif min_depth and pdu.depth > min_depth and max_recursion > 0: - for event_id, hashes in pdu.prev_events: - if event_id not in have_seen: - logger.debug( - "_handle_new_pdu requesting pdu %s", - event_id + elif min_depth and pdu.depth > min_depth: + if get_missing and prevs - seen: + latest_tuples = yield self.store.get_latest_events_in_room( + pdu.room_id + ) + + # We add the prev events that we have seen to the latest + # list to ensure the remote server doesn't give them to us + latest = set(e_id for e_id, _, _ in latest_tuples) + latest |= seen + + missing_events = yield self.get_missing_events( + origin, + pdu.room_id, + earliest_events=list(latest), + latest_events=[pdu.event_id], + limit=10, + min_depth=min_depth, + ) + + for e in missing_events: + yield self._handle_new_pdu( + origin, + e, + get_missing=False ) - try: - new_pdu = yield self.federation_client.get_pdu( - [origin, pdu.origin], - event_id=event_id, - ) - - if new_pdu: - yield self._handle_new_pdu( - origin, - new_pdu, - max_recursion=max_recursion-1 - ) - - logger.debug("Processed pdu %s", event_id) - else: - logger.warn("Failed to get PDU %s", event_id) - fetch_state = True - except: - # TODO(erikj): Do some more intelligent retries. - logger.exception("Failed to get PDU") - fetch_state = True - else: - prevs = {e_id for e_id, _ in pdu.prev_events} - seen = set(have_seen.keys()) - if prevs - seen: - fetch_state = True - else: - fetch_state = True + have_seen = yield self.store.have_events( + [ev for ev, _ in pdu.prev_events] + ) + + prevs = {e_id for e_id, _ in pdu.prev_events} + seen = set(have_seen.keys()) + if prevs - seen: + fetch_state = True if fetch_state: # We need to get the state at this event, since we haven't diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 7d30c924d1..8f1acbe590 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -287,7 +287,7 @@ class TransactionQueue(object): code = 200 if response: - for e_id, r in getattr(response, "pdus", {}).items(): + for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warn( "Transaction returned error for %s: %s", diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 8b137e7128..80d03012b7 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -219,3 +219,22 @@ class TransportLayerClient(object): ) defer.returnValue(content) + + @defer.inlineCallbacks + @log_function + def get_missing_events(self, destination, room_id, earliest_events, + latest_events, limit, min_depth): + path = PREFIX + "/get_missing_events/%s" % (room_id,) + + content = yield self.client.post_json( + destination=destination, + path=path, + data={ + "limit": int(limit), + "min_depth": int(min_depth), + "earliest_events": earliest_events, + "latest_events": latest_events, + } + ) + + defer.returnValue(content) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 2ffb37aa18..ad75c8ddb7 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -234,6 +234,7 @@ class TransportLayerServer(object): ) ) ) + self.server.register_path( "POST", re.compile("^" + PREFIX + "/query_auth/([^/]*)/([^/]*)$"), @@ -245,6 +246,17 @@ class TransportLayerServer(object): ) ) + self.server.register_path( + "POST", + re.compile("^" + PREFIX + "/get_missing_events/([^/]*)/?$"), + self._with_authentication( + lambda origin, content, query, room_id: + self._get_missing_events( + origin, content, room_id, + ) + ) + ) + @defer.inlineCallbacks @log_function def _on_send_request(self, origin, content, query, transaction_id): @@ -344,3 +356,22 @@ class TransportLayerServer(object): ) defer.returnValue((200, new_content)) + + @defer.inlineCallbacks + @log_function + def _get_missing_events(self, origin, content, room_id): + limit = int(content.get("limit", 10)) + min_depth = int(content.get("min_depth", 0)) + earliest_events = content.get("earliest_events", []) + latest_events = content.get("latest_events", []) + + content = yield self.request_handler.on_get_missing_events( + origin, + room_id=room_id, + earliest_events=earliest_events, + latest_events=latest_events, + min_depth=min_depth, + limit=limit, + ) + + defer.returnValue((200, content)) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 26bdc6d1a7..628e62f8b1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -789,6 +789,29 @@ class FederationHandler(BaseHandler): defer.returnValue(ret) + @defer.inlineCallbacks + def on_get_missing_events(self, origin, room_id, earliest_events, + latest_events, limit, min_depth): + in_room = yield self.auth.check_host_in_room( + room_id, + origin + ) + if not in_room: + raise AuthError(403, "Host not in room.") + + limit = min(limit, 20) + min_depth = max(min_depth, 0) + + missing_events = yield self.store.get_missing_events( + room_id=room_id, + earliest_events=earliest_events, + latest_events=latest_events, + limit=limit, + min_depth=min_depth, + ) + + defer.returnValue(missing_events) + @defer.inlineCallbacks @log_function def do_auth(self, origin, event, context, auth_events): -- cgit 1.4.1 From 42b972bccd0cf7d903befb498f9c1bbd5c4e6583 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Feb 2015 14:35:23 +0000 Subject: Revert get_auth_chain changes --- synapse/storage/event_federation.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 22bf7ad832..2deda8ac50 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -32,15 +32,15 @@ class EventFederationStore(SQLBaseStore): and backfilling from another server respectively. """ - def get_auth_chain(self, event_ids, have_ids=set()): + def get_auth_chain(self, event_ids): return self.runInteraction( "get_auth_chain", self._get_auth_chain_txn, - event_ids, have_ids + event_ids ) - def _get_auth_chain_txn(self, txn, event_ids, have_ids): - results = self._get_auth_chain_ids_txn(txn, event_ids, have_ids) + def _get_auth_chain_txn(self, txn, event_ids): + results = self._get_auth_chain_ids_txn(txn, event_ids) return self._get_events_txn(txn, results) @@ -51,9 +51,8 @@ class EventFederationStore(SQLBaseStore): event_ids ) - def _get_auth_chain_ids_txn(self, txn, event_ids, have_ids): + def _get_auth_chain_ids_txn(self, txn, event_ids): results = set() - have_ids = set(have_ids) base_sql = ( "SELECT auth_id FROM event_auth WHERE event_id = ?" @@ -67,7 +66,6 @@ class EventFederationStore(SQLBaseStore): new_front.update([r[0] for r in txn.fetchall()]) new_front -= results - new_front -= have_ids front = new_front results.update(front) -- cgit 1.4.1 From 4631b737fdb08185d514e69f0e6860c0860768b5 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 14:38:44 +0000 Subject: Squash out the now-redundant ApplicationServicesCache object class --- synapse/storage/appservice.py | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index d941b1f387..dc3666efd4 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -23,23 +23,11 @@ from ._base import SQLBaseStore logger = logging.getLogger(__name__) -class ApplicationServiceCache(object): - """Caches ApplicationServices and provides utility functions on top. - - This class is designed to be invoked on incoming events in order to avoid - hammering the database every time to extract a list of application service - regexes. - """ - - def __init__(self): - self.services = [] - - class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) - self.cache = ApplicationServiceCache() + self.services_cache = [] self.cache_defer = self._populate_cache() @defer.inlineCallbacks @@ -56,7 +44,7 @@ class ApplicationServiceStore(SQLBaseStore): token, ) # update cache TODO: Should this be in the txn? - for service in self.cache.services: + for service in self.services_cache: if service.token == token: service.url = None service.namespaces = None @@ -110,13 +98,13 @@ class ApplicationServiceStore(SQLBaseStore): ) # update cache TODO: Should this be in the txn? - for (index, cache_service) in enumerate(self.cache.services): + for (index, cache_service) in enumerate(self.services_cache): if service.token == cache_service.token: - self.cache.services[index] = service + self.services_cache[index] = service logger.info("Updated: %s", service) return # new entry - self.cache.services.append(service) + self.services_cache.append(service) logger.info("Updated(new): %s", service) def _update_app_service_txn(self, txn, service): @@ -160,7 +148,7 @@ class ApplicationServiceStore(SQLBaseStore): @defer.inlineCallbacks def get_app_services(self): yield self.cache_defer # make sure the cache is ready - defer.returnValue(self.cache.services) + defer.returnValue(self.services_cache) @defer.inlineCallbacks def get_app_service_by_token(self, token, from_cache=True): @@ -176,7 +164,7 @@ class ApplicationServiceStore(SQLBaseStore): yield self.cache_defer # make sure the cache is ready if from_cache: - for service in self.cache.services: + for service in self.services_cache: if service.token == token: defer.returnValue(service) return @@ -235,7 +223,7 @@ class ApplicationServiceStore(SQLBaseStore): # TODO get last successful txn id f.e. service for service in services.values(): logger.info("Found application service: %s", service) - self.cache.services.append(ApplicationService( + self.services_cache.append(ApplicationService( token=service["token"], url=service["url"], namespaces=service["namespaces"], -- cgit 1.4.1 From 22399d3d8f5931c4302a9df37ebde94f5d1fc277 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 23 Feb 2015 15:14:56 +0000 Subject: Add RegisterFallbackResource to /_matrix/static/client/register Try to keep both forms of registration logic (native/fallback) close together for sanity. --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 2 ++ synapse/rest/client/v1/__init__.py | 8 ++++++++ synapse/rest/client/v1/register.py | 21 +++++++++++++++++++++ 4 files changed, 32 insertions(+) (limited to 'synapse') diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 9485719332..3d43674625 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -18,6 +18,7 @@ CLIENT_PREFIX = "/_matrix/client/api/v1" CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha" FEDERATION_PREFIX = "/_matrix/federation/v1" +STATIC_PREFIX = "/_matrix/static" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_PREFIX = "/_matrix/key/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ea20de1434..137293bd69 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -125,6 +125,8 @@ class SynapseHomeServer(HomeServer): (MEDIA_PREFIX, self.get_resource_for_media_repository()), (APP_SERVICE_PREFIX, self.get_resource_for_app_services()), ] + desired_tree += self.get_resource_for_client().get_extra_resources(self) + if web_client: logger.info("Adding the web client.") desired_tree.append((WEB_CLIENT_PREFIX, diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index 21876b3487..1ebdeadca5 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -28,6 +28,14 @@ class ClientV1RestResource(JsonResource): JsonResource.__init__(self, hs) self.register_servlets(self, hs) + def get_extra_resources(self, hs): + # some parts of client v1 need to produce HTML as the output (e.g. + # fallback pages) but we can only produce JSON output. In an effort to + # keep similar logic close together, we'll call through to any servlet + # which requires HTML output. + register_resources = register.get_prefixes_and_resources(hs) + return register_resources + @staticmethod def register_servlets(client_resource, hs): room.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index f5acfb945f..eee567a583 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -18,10 +18,12 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes from synapse.api.constants import LoginType +from synapse.api.urls import STATIC_PREFIX from base import ClientV1RestServlet, client_path_pattern import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor +from twisted.web.resource import Resource from hashlib import sha1 import hmac @@ -305,6 +307,16 @@ class RegisterRestServlet(ClientV1RestServlet): }) +class RegisterFallbackResource(Resource): + + def __init__(self, hs): + Resource.__init__(self) # Resource is an old-style class :( + self.hs = hs + + def render_GET(self, request): + return "NOT_YET_IMPLEMENTED" + + def _parse_json(request): try: content = json.loads(request.content.read()) @@ -315,5 +327,14 @@ def _parse_json(request): raise SynapseError(400, "Content not JSON.") +def get_prefixes_and_resources(hs): + return [ + ( + STATIC_PREFIX + "/client/register", + RegisterFallbackResource(hs) + ) + ] + + def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) -- cgit 1.4.1 From 0696dfd94b5d32679009368508d26ae1a6630994 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 23 Feb 2015 15:35:09 +0000 Subject: Actually treat this as static content, not random Resources. --- synapse/app/homeserver.py | 8 ++++++-- synapse/rest/client/v1/__init__.py | 8 -------- synapse/rest/client/v1/register.py | 21 --------------------- synapse/server.py | 1 + 4 files changed, 7 insertions(+), 31 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 137293bd69..688bf8d967 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -36,7 +36,8 @@ from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, APP_SERVICE_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, APP_SERVICE_PREFIX, + STATIC_PREFIX ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory @@ -81,6 +82,9 @@ class SynapseHomeServer(HomeServer): webclient_path = os.path.join(syweb_path, "webclient") return File(webclient_path) # TODO configurable? + def build_resource_for_static_content(self): + return File("static") + def build_resource_for_content_repo(self): return ContentRepoResource( self, self.upload_dir, self.auth, self.content_addr @@ -124,8 +128,8 @@ class SynapseHomeServer(HomeServer): (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), (MEDIA_PREFIX, self.get_resource_for_media_repository()), (APP_SERVICE_PREFIX, self.get_resource_for_app_services()), + (STATIC_PREFIX, self.get_resource_for_static_content()) ] - desired_tree += self.get_resource_for_client().get_extra_resources(self) if web_client: logger.info("Adding the web client.") diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index 1ebdeadca5..21876b3487 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -28,14 +28,6 @@ class ClientV1RestResource(JsonResource): JsonResource.__init__(self, hs) self.register_servlets(self, hs) - def get_extra_resources(self, hs): - # some parts of client v1 need to produce HTML as the output (e.g. - # fallback pages) but we can only produce JSON output. In an effort to - # keep similar logic close together, we'll call through to any servlet - # which requires HTML output. - register_resources = register.get_prefixes_and_resources(hs) - return register_resources - @staticmethod def register_servlets(client_resource, hs): room.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index eee567a583..f5acfb945f 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -18,12 +18,10 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes from synapse.api.constants import LoginType -from synapse.api.urls import STATIC_PREFIX from base import ClientV1RestServlet, client_path_pattern import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor -from twisted.web.resource import Resource from hashlib import sha1 import hmac @@ -307,16 +305,6 @@ class RegisterRestServlet(ClientV1RestServlet): }) -class RegisterFallbackResource(Resource): - - def __init__(self, hs): - Resource.__init__(self) # Resource is an old-style class :( - self.hs = hs - - def render_GET(self, request): - return "NOT_YET_IMPLEMENTED" - - def _parse_json(request): try: content = json.loads(request.content.read()) @@ -327,14 +315,5 @@ def _parse_json(request): raise SynapseError(400, "Content not JSON.") -def get_prefixes_and_resources(hs): - return [ - ( - STATIC_PREFIX + "/client/register", - RegisterFallbackResource(hs) - ) - ] - - def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index ba2b2593f1..cb8610a1b4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -73,6 +73,7 @@ class BaseHomeServer(object): 'resource_for_client', 'resource_for_client_v2_alpha', 'resource_for_federation', + 'resource_for_static_content', 'resource_for_web_client', 'resource_for_content_repo', 'resource_for_server_key', -- cgit 1.4.1 From e76d485e29498fce7412423e7a5b6ac6bc287ec3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 15:41:54 +0000 Subject: Allow @cached-wrapped functions to have a prefill method for setting entries --- synapse/storage/_base.py | 23 +++++++++++++++-------- tests/storage/test__base.py | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 78ba5f25ea..4b1ec687c9 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -48,24 +48,30 @@ def cached(max_entries=1000): The wrapped function has an additional member, a callable called "invalidate". This can be used to remove individual entries from the cache. + + The wrapped function has another additional callable, called "prefill", + which can be used to insert values into the cache specifically, without + calling the calculation function. """ def wrap(orig): cache = {} - @defer.inlineCallbacks - def wrapped(self, key): - if key in cache: - defer.returnValue(cache[key]) - - ret = yield orig(self, key) - + def prefill(key, value): while len(cache) > max_entries: # TODO(paul): This feels too biased. However, a random index # would be a bit inefficient, walking the list of keys just # to ignore most of them? del cache[cache.keys()[0]] - cache[key] = ret; + cache[key] = value + + @defer.inlineCallbacks + def wrapped(self, key): + if key in cache: + defer.returnValue(cache[key]) + + ret = yield orig(self, key) + prefill(key, ret) defer.returnValue(ret) def invalidate(key): @@ -73,6 +79,7 @@ def cached(max_entries=1000): del cache[key] wrapped.invalidate = invalidate + wrapped.prefill = prefill return wrapped return wrap diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index 057f798640..fb306cb784 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -87,3 +87,17 @@ class CacheDecoratorTestCase(unittest.TestCase): self.assertTrue(callcount[0] >= 14, msg="Expected callcount >= 14, got %d" % (callcount[0])) + + @defer.inlineCallbacks + def test_prefill(self): + callcount = [0] + + @cached() + def func(self, key): + callcount[0] += 1 + return key + + func.prefill("foo", 123) + + self.assertEquals((yield func(self, "foo")), 123) + self.assertEquals(callcount[0], 0) -- cgit 1.4.1 From 357fba2c24067796ce89f25636a2541bc9a10752 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 15:57:41 +0000 Subject: RoomMemberStore no longer needs a _user_rooms_cache member --- synapse/storage/roommember.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 33a832483e..58aa376c20 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -35,11 +35,6 @@ RoomsForUser = namedtuple( class RoomMemberStore(SQLBaseStore): - def __init__(self, *args, **kw): - super(RoomMemberStore, self).__init__(*args, **kw) - - self._user_rooms_cache = {} - def _store_room_member_txn(self, txn, event): """Store a room member in the database. """ -- cgit 1.4.1 From 044d813ef7bb67e7b680087d82f931e4c780218f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 16:04:40 +0000 Subject: Use the @cached decorator to implement the destination_retry_timings cache --- synapse/storage/transactions.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index e06ef35690..6cac8d01ac 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, Table +from ._base import SQLBaseStore, Table, cached from collections import namedtuple @@ -28,10 +28,6 @@ class TransactionStore(SQLBaseStore): """A collection of queries for handling PDUs. """ - # a write-through cache of DestinationsTable.EntryType indexed by - # destination string - destination_retry_cache = {} - def get_received_txn_response(self, transaction_id, origin): """For an incoming transaction from a given origin, check if we have already responded to it. If so, return the response code and response @@ -211,6 +207,7 @@ class TransactionStore(SQLBaseStore): return ReceivedTransactionsTable.decode_results(txn.fetchall()) + @cached() def get_destination_retry_timings(self, destination): """Gets the current retry timings (if any) for a given destination. @@ -221,9 +218,6 @@ class TransactionStore(SQLBaseStore): None if not retrying Otherwise a DestinationsTable.EntryType for the retry scheme """ - if destination in self.destination_retry_cache: - return defer.succeed(self.destination_retry_cache[destination]) - return self.runInteraction( "get_destination_retry_timings", self._get_destination_retry_timings, destination) @@ -250,7 +244,9 @@ class TransactionStore(SQLBaseStore): retry_interval (int) - how long until next retry in ms """ - self.destination_retry_cache[destination] = ( + # As this is the new value, we might as well prefill the cache + self.get_destination_retry_timings.prefill( + destination, DestinationsTable.EntryType( destination, retry_last_ts, -- cgit 1.4.1 From a09e59a69830ba99d65f54b2385c3cab341accb0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 16:55:57 +0000 Subject: Pull the _get_event_cache.setdefault() call out of the try block, as it doesn't need to be there and is confusing --- synapse/storage/_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4b1ec687c9..84f222b3db 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -636,8 +636,9 @@ class SQLBaseStore(object): start_time = time.time() * 1000 update_counter = self._get_event_counters.update + cache = self._get_event_cache.setdefault(event_id, {}) + try: - cache = self._get_event_cache.setdefault(event_id, {}) # Separate cache entries for each way to invoke _get_event_txn return cache[(check_redacted, get_prev_content, allow_rejected)] except KeyError: -- cgit 1.4.1 From 28d8614f4879a4e5276c1c14190ad1b4ca069350 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 23 Feb 2015 17:36:37 +0000 Subject: Trailing comma --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 688bf8d967..2d6f6ac89d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -128,7 +128,7 @@ class SynapseHomeServer(HomeServer): (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), (MEDIA_PREFIX, self.get_resource_for_media_repository()), (APP_SERVICE_PREFIX, self.get_resource_for_app_services()), - (STATIC_PREFIX, self.get_resource_for_static_content()) + (STATIC_PREFIX, self.get_resource_for_static_content()), ] if web_client: -- cgit 1.4.1 From 74048bdd41108e0c98e034b6ebc5890fd3bcf92b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 23 Feb 2015 18:17:43 +0000 Subject: Remove unused import --- synapse/storage/transactions.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 6cac8d01ac..0b8a3b7a07 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -17,8 +17,6 @@ from ._base import SQLBaseStore, Table, cached from collections import namedtuple -from twisted.internet import defer - import logging logger = logging.getLogger(__name__) -- cgit 1.4.1 From 27080698e7e9c7d8bb8dfe473f888abdfc48fee7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 23 Feb 2015 18:19:13 +0000 Subject: Fix code style warning --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 84f222b3db..42edb56c36 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -79,7 +79,7 @@ def cached(max_entries=1000): del cache[key] wrapped.invalidate = invalidate - wrapped.prefill = prefill + wrapped.prefill = prefill return wrapped return wrap -- cgit 1.4.1 From f53fcbce9789186c1c42fe2f93ba46e3d8720b1b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 18:29:26 +0000 Subject: Use cache.pop() instead of a separate membership test + del [] --- synapse/storage/_base.py | 3 +-- tests/storage/test__base.py | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 42edb56c36..da698cb3b8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -75,8 +75,7 @@ def cached(max_entries=1000): defer.returnValue(ret) def invalidate(key): - if key in cache: - del cache[key] + cache.pop(key, None) wrapped.invalidate = invalidate wrapped.prefill = prefill diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index fb306cb784..55d22f665a 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -66,6 +66,13 @@ class CacheDecoratorTestCase(unittest.TestCase): self.assertEquals(callcount[0], 2) + def test_invalidate_missing(self): + @cached() + def func(self, key): + return key + + func.invalidate("what") + @defer.inlineCallbacks def test_max_entries(self): callcount = [0] -- cgit 1.4.1 From 9640510de206d673e4c0c9cbd1cef219bcd488b2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 18:41:58 +0000 Subject: Use OrderedDict for @cached backing store, so we can evict the oldest key unbiased --- synapse/storage/_base.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index da698cb3b8..c98dd36aed 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -23,7 +23,7 @@ from synapse.util.lrucache import LruCache from twisted.internet import defer -import collections +from collections import namedtuple, OrderedDict import simplejson as json import sys import time @@ -54,14 +54,11 @@ def cached(max_entries=1000): calling the calculation function. """ def wrap(orig): - cache = {} + cache = OrderedDict() def prefill(key, value): while len(cache) > max_entries: - # TODO(paul): This feels too biased. However, a random index - # would be a bit inefficient, walking the list of keys just - # to ignore most of them? - del cache[cache.keys()[0]] + cache.popitem(last=False) cache[key] = value @@ -836,7 +833,7 @@ class JoinHelper(object): for table in self.tables: res += [f for f in table.fields if f not in res] - self.EntryType = collections.namedtuple("JoinHelperEntry", res) + self.EntryType = namedtuple("JoinHelperEntry", res) def get_fields(self, **prefixes): """Get a string representing a list of fields for use in SELECT -- cgit 1.4.1 From 443ba4eecc59e3dbddfaf5b925408b40e3581800 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Feb 2015 15:00:12 +0000 Subject: %s for strings otherwise you end up sending 'u"foo"' --- synapse/handlers/directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 20ab9e269c..11d20a5d2d 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -160,7 +160,7 @@ class DirectoryHandler(BaseHandler): if not room_id: raise SynapseError( 404, - "Room alias %r not found" % (room_alias.to_string(),), + "Room alias %s not found" % (room_alias.to_string(),), Codes.NOT_FOUND ) -- cgit 1.4.1 From 255f989c7bc20d17ec37fed00cb2d107553d6f73 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Feb 2015 20:57:58 +0000 Subject: turns uris config options should append since it's a list --- synapse/config/voip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/config/voip.py b/synapse/config/voip.py index a2b822719f..65162d21b7 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -28,7 +28,7 @@ class VoipConfig(Config): super(VoipConfig, cls).add_arguments(parser) group = parser.add_argument_group("voip") group.add_argument( - "--turn-uris", type=str, default=None, + "--turn-uris", type=str, default=None, action='append', help="The public URIs of the TURN server to give to clients" ) group.add_argument( -- cgit 1.4.1 From a025055643638ddb612ac16cf8deff8ae19ab1e0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 25 Feb 2015 14:02:38 +0000 Subject: SYWEB-278 Don't allow rules with no rule_id. --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index b012f31084..73ba0494e6 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -214,7 +214,7 @@ def _rule_spec_from_path(path): template = path[0] path = path[1:] - if len(path) == 0: + if len(path) == 0 or len(path[0]) == 0: raise UnrecognizedRequestError() rule_id = path[0] -- cgit 1.4.1 From 2d20466f9a1349c97d5a3822eb4ee64f19bbdf27 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 25 Feb 2015 15:00:59 +0000 Subject: Add stub functions and work out execution flow to implement AS event stream polling. --- synapse/handlers/events.py | 3 --- synapse/handlers/room.py | 34 +++++++++++++++++++++++++--------- synapse/storage/appservice.py | 19 +++++++++++++++++++ synapse/storage/stream.py | 21 +++++++++++++++++++++ 4 files changed, 65 insertions(+), 12 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 025e7e7e62..8d5f5c8499 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -69,9 +69,6 @@ class EventStreamHandler(BaseHandler): ) self._streams_per_user[auth_user] += 1 - if pagin_config.from_token is None: - pagin_config.from_token = None - rm_handler = self.hs.get_handlers().room_member_handler room_ids = yield rm_handler.get_rooms_for_user(auth_user) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 914742d913..a8b0c95636 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -510,9 +510,16 @@ class RoomMemberHandler(BaseHandler): def get_rooms_for_user(self, user, membership_list=[Membership.JOIN]): """Returns a list of roomids that the user has any of the given membership states in.""" - rooms = yield self.store.get_rooms_for_user_where_membership_is( - user_id=user.to_string(), membership_list=membership_list + + app_service = yield self.store.get_app_service_by_user_id( + user.to_string() ) + if app_service: + rooms = yield self.store.get_app_service_rooms(app_service) + else: + rooms = yield self.store.get_rooms_for_user_where_membership_is( + user_id=user.to_string(), membership_list=membership_list + ) # For some reason the list of events contains duplicates # TODO(paul): work out why because I really don't think it should @@ -559,13 +566,22 @@ class RoomEventSource(object): to_key = yield self.get_current_key() - events, end_key = yield self.store.get_room_events_stream( - user_id=user.to_string(), - from_key=from_key, - to_key=to_key, - room_id=None, - limit=limit, - ) + app_service = self.store.get_app_service_by_user_id(user.to_string()) + if app_service: + events, end_key = yield self.store.get_appservice_room_stream( + service=app_service, + from_key=from_key, + to_key=to_key, + limit=limit, + ) + else: + events, end_key = yield self.store.get_room_events_stream( + user_id=user.to_string(), + from_key=from_key, + to_key=to_key, + room_id=None, + limit=limit, + ) defer.returnValue((events, end_key)) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index dc3666efd4..435ccfd6fc 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -17,6 +17,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError from synapse.appservice import ApplicationService +from synapse.storage.roommember import RoomsForUser from ._base import SQLBaseStore @@ -150,6 +151,16 @@ class ApplicationServiceStore(SQLBaseStore): yield self.cache_defer # make sure the cache is ready defer.returnValue(self.services_cache) + @defer.inlineCallbacks + def get_app_service_by_user_id(self, user_id): + yield self.cache_defer # make sure the cache is ready + + for service in self.services_cache: + if service.sender == user_id: + defer.returnValue(service) + return + defer.returnValue(None) + @defer.inlineCallbacks def get_app_service_by_token(self, token, from_cache=True): """Get the application service with the given token. @@ -173,6 +184,14 @@ class ApplicationServiceStore(SQLBaseStore): # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. + @defer.inlineCallbacks + def get_app_service_rooms(self, service): + logger.info("get_app_service_rooms -> %s", service) + + # TODO stub + yield self.cache_defer + defer.returnValue([RoomsForUser("!foo:bar", service.sender, "join")]) + @defer.inlineCallbacks def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3ccb6f8a61..aa3c9f8c9c 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -127,6 +127,27 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): class StreamStore(SQLBaseStore): + + def get_appservice_room_stream(self, service, from_key, to_key, limit=0): + # NB this lives here instead of appservice.py so we can reuse the + # 'private' StreamToken class in this file. + logger.info("get_appservice_room_stream -> %s", service) + + if limit: + limit = max(limit, MAX_STREAM_SIZE) + else: + limit = MAX_STREAM_SIZE + + # From and to keys should be integers from ordering. + from_id = _StreamToken.parse_stream_token(from_key) + to_id = _StreamToken.parse_stream_token(to_key) + + if from_key == to_key: + return defer.succeed(([], to_key)) + + # TODO stub + return defer.succeed(([], to_key)) + @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, limit=0, with_feedback=False): -- cgit 1.4.1 From 2b8ca84296b228b7cef09244605e4f2760349538 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 25 Feb 2015 17:15:25 +0000 Subject: Add support for extracting matching room_ids and room_aliases for a given AS. --- synapse/storage/appservice.py | 50 +++++++++++++++++++++++++++++++++++++++++-- synapse/storage/directory.py | 23 ++++++++++++++++++++ synapse/storage/room.py | 11 ++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 435ccfd6fc..c8f0ce44f4 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -153,6 +153,19 @@ class ApplicationServiceStore(SQLBaseStore): @defer.inlineCallbacks def get_app_service_by_user_id(self, user_id): + """Retrieve an application service from their user ID. + + All application services have associated with them a particular user ID. + There is no distinguishing feature on the user ID which indicates it + represents an application service. This function allows you to map from + a user ID to an application service. + + Args: + user_id(str): The user ID to see if it is an application service. + Returns: + synapse.appservice.ApplicationService or None. + """ + yield self.cache_defer # make sure the cache is ready for service in self.services_cache: @@ -163,7 +176,7 @@ class ApplicationServiceStore(SQLBaseStore): @defer.inlineCallbacks def get_app_service_by_token(self, token, from_cache=True): - """Get the application service with the given token. + """Get the application service with the given appservice token. Args: token (str): The application service token. @@ -186,10 +199,43 @@ class ApplicationServiceStore(SQLBaseStore): @defer.inlineCallbacks def get_app_service_rooms(self, service): - logger.info("get_app_service_rooms -> %s", service) + """Get a list of RoomsForUser for this application service. + + Application services may be "interested" in lots of rooms depending on + the room ID, the room aliases, or the members in the room. This function + takes all of these into account and returns a list of RoomsForUser which + represent the entire list of room IDs that this application service + wants to know about. + + Args: + service: The application service to get a room list for. + Returns: + A list of RoomsForUser. + """ + # FIXME: This is assuming that this store has methods from + # RoomStore, DirectoryStore, which is a bad assumption to + # make as it makes testing trickier and coupling less obvious. + + # get all rooms matching the room ID regex. + room_entries = yield self.get_all_rooms() # RoomEntry list + matching_room_id_list = [ + r.room_id for r in room_entries if + service.is_interested_in_room(r.room_id) + ] + + # resolve room IDs for matching room alias regex. + room_alias_mappings = yield self.get_all_associations() + matching_alias_list = [ + r.room_id for r in room_alias_mappings if + service.is_interested_in_alias(r.room_alias) + ] + + # get all rooms for every user for this AS. # TODO stub yield self.cache_defer + + defer.returnValue([RoomsForUser("!foo:bar", service.sender, "join")]) @defer.inlineCallbacks diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 68b7d59693..e13b336934 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -134,6 +134,29 @@ class DirectoryStore(SQLBaseStore): return room_id + @defer.inlineCallbacks + def get_all_associations(self): + """Retrieve the entire list of room alias -> room ID pairings. + + Returns: + A list of RoomAliasMappings. + """ + results = self._simple_select_list( + "room_aliases", + None, + ["room_alias", "room_id"] + ) + # TODO(kegan): It feels wrong to be specifying no servers here, but + # equally this function isn't required to obtain all servers so + # retrieving them "just for the sake of it" also seems wrong, but we + # want to conform to passing Objects around and not dicts.. + return [ + RoomAliasMapping( + room_id=r["room_id"], room_alias=r["room_alias"], servers="" + ) for r in results + ] + + def get_aliases_for_room(self, room_id): return self._simple_select_onecol( "room_aliases", diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 750b17a45f..3a64693404 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -71,6 +71,17 @@ class RoomStore(SQLBaseStore): RoomsTable.decode_single_result, query, room_id, ) + def get_all_rooms(self): + """Retrieve all the rooms. + + Returns: + A list of namedtuples containing the room information. + """ + query = RoomsTable.select_statement() + return self._execute( + RoomsTable.decode_results, query, + ) + @defer.inlineCallbacks def get_rooms(self, is_public): """Retrieve a list of all public rooms. -- cgit 1.4.1 From 2c79c4dc7f638f1cb823903a2f8bb1005fda4a2c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 25 Feb 2015 17:37:14 +0000 Subject: Fix alias query. --- synapse/storage/directory.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index e13b336934..70c8c8ccd3 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -141,20 +141,19 @@ class DirectoryStore(SQLBaseStore): Returns: A list of RoomAliasMappings. """ - results = self._simple_select_list( - "room_aliases", - None, - ["room_alias", "room_id"] + results = yield self._execute_and_decode( + "SELECT room_id, room_alias FROM room_aliases" ) + # TODO(kegan): It feels wrong to be specifying no servers here, but # equally this function isn't required to obtain all servers so # retrieving them "just for the sake of it" also seems wrong, but we # want to conform to passing Objects around and not dicts.. - return [ + defer.returnValue([ RoomAliasMapping( room_id=r["room_id"], room_alias=r["room_alias"], servers="" ) for r in results - ] + ]) def get_aliases_for_room(self, room_id): -- cgit 1.4.1 From 978ce87c86e60fd49f078d5bea79715abea6d236 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 25 Feb 2015 17:37:48 +0000 Subject: Comment unused variables. --- synapse/storage/stream.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index aa3c9f8c9c..3c8f3320f1 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -139,8 +139,8 @@ class StreamStore(SQLBaseStore): limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. - from_id = _StreamToken.parse_stream_token(from_key) - to_id = _StreamToken.parse_stream_token(to_key) + # from_id = _StreamToken.parse_stream_token(from_key) + # to_id = _StreamToken.parse_stream_token(to_key) if from_key == to_key: return defer.succeed(([], to_key)) -- cgit 1.4.1 From 29267cf9d7fbacdfcccaaef9160657f24b9aca14 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 25 Feb 2015 17:42:28 +0000 Subject: PEP8 and pyflakes --- synapse/storage/appservice.py | 8 +++----- synapse/storage/directory.py | 1 - 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index c8f0ce44f4..017b6d1e86 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -229,12 +229,10 @@ class ApplicationServiceStore(SQLBaseStore): r.room_id for r in room_alias_mappings if service.is_interested_in_alias(r.room_alias) ] + logging.debug(matching_alias_list) + logging.debug(matching_room_id_list) - # get all rooms for every user for this AS. - - # TODO stub - yield self.cache_defer - + # TODO get all rooms for every user for this AS. defer.returnValue([RoomsForUser("!foo:bar", service.sender, "join")]) diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 70c8c8ccd3..e391239a3c 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -155,7 +155,6 @@ class DirectoryStore(SQLBaseStore): ) for r in results ]) - def get_aliases_for_room(self, room_id): return self._simple_select_onecol( "room_aliases", -- cgit 1.4.1 From 94fa334b01d232bed96fd4ee05fc44d00330c2b9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 25 Feb 2015 19:17:07 +0000 Subject: Add enable/disable overlay for push rules (REST API not yet hooked up) --- synapse/push/__init__.py | 4 ++++ synapse/push/baserules.py | 3 +++ synapse/storage/push_rule.py | 20 ++++++++++++++++++++ synapse/storage/schema/delta/next_pushrules2.sql | 9 +++++++++ synapse/storage/schema/pusher.sql | 10 ++++++++++ 5 files changed, 46 insertions(+) create mode 100644 synapse/storage/schema/delta/next_pushrules2.sql (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 0fb3e4f7f3..40fae91ab5 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -82,6 +82,8 @@ class Pusher(object): r['conditions'] = json.loads(r['conditions']) r['actions'] = json.loads(r['actions']) + enabled_map = yield self.store.get_push_rules_enabled_for_user_name(self.user_name) + user = UserID.from_string(self.user_name) rules = baserules.list_with_base_rules(rawrules, user) @@ -107,6 +109,8 @@ class Pusher(object): room_member_count += 1 for r in rules: + if r['rule_id'] in enabled_map and not enabled_map[r['rule_id']]: + continue matches = True conditions = r['conditions'] diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 162d265f66..ba9a181b56 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -45,6 +45,7 @@ def make_base_rules(user, kind): def make_base_content_rules(user): return [ { + 'rule_id': '.m.rule.contains_user_name', 'conditions': [ { 'kind': 'event_match', @@ -66,6 +67,7 @@ def make_base_content_rules(user): def make_base_override_rules(): return [ { + 'rule_id': '.m.rule.contains_display_name', 'conditions': [ { 'kind': 'contains_display_name' @@ -80,6 +82,7 @@ def make_base_override_rules(): ] }, { + 'rule_id': '.m.rule.room_two_members', 'conditions': [ { 'kind': 'room_member_count', diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index ae46b39cc1..3890c7aab2 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -45,6 +45,17 @@ class PushRuleStore(SQLBaseStore): defer.returnValue(dicts) + @defer.inlineCallbacks + def get_push_rules_enabled_for_user_name(self, user_name): + results = yield self._simple_select_list( + PushRuleEnableTable.table_name, + {'user_name': user_name}, + PushRuleEnableTable.fields + ) + defer.returnValue( + {r['rule_id']: False if r['enabled'] == 0 else True for r in results} + ) + @defer.inlineCallbacks def add_push_rule(self, before, after, **kwargs): vals = copy.copy(kwargs) @@ -216,3 +227,12 @@ class PushRuleTable(Table): ] EntryType = collections.namedtuple("PushRuleEntry", fields) + +class PushRuleEnableTable(Table): + table_name = "push_rules_enable" + + fields = [ + "user_name", + "rule_id", + "enabled" + ] \ No newline at end of file diff --git a/synapse/storage/schema/delta/next_pushrules2.sql b/synapse/storage/schema/delta/next_pushrules2.sql new file mode 100644 index 0000000000..0212726448 --- /dev/null +++ b/synapse/storage/schema/delta/next_pushrules2.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS push_rules_enable ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + enabled TINYINT, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 3735b11547..31bf1cb685 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -44,3 +44,13 @@ CREATE TABLE IF NOT EXISTS push_rules ( ); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); + +CREATE TABLE IF NOT EXISTS push_rules_enable ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + enabled TINYINT, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); -- cgit 1.4.1 From 944003021bb9f6aa232d1436761c1c07f0273241 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 26 Feb 2015 13:43:05 +0000 Subject: whitespace --- synapse/storage/push_rule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 3890c7aab2..cd8d0f6dde 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -228,6 +228,7 @@ class PushRuleTable(Table): EntryType = collections.namedtuple("PushRuleEntry", fields) + class PushRuleEnableTable(Table): table_name = "push_rules_enable" @@ -235,4 +236,4 @@ class PushRuleEnableTable(Table): "user_name", "rule_id", "enabled" - ] \ No newline at end of file + ] -- cgit 1.4.1 From 92478e96d6f6992146102599ca96b8dcacbf3895 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 26 Feb 2015 14:35:28 +0000 Subject: Finish impl to extract all room IDs an AS may be interested in when polling the event stream. --- synapse/storage/appservice.py | 35 +++++++++++++++++++++++++++++------ synapse/storage/registration.py | 7 +++++++ 2 files changed, 36 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 017b6d1e86..d0632d55d1 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -213,8 +213,9 @@ class ApplicationServiceStore(SQLBaseStore): A list of RoomsForUser. """ # FIXME: This is assuming that this store has methods from - # RoomStore, DirectoryStore, which is a bad assumption to - # make as it makes testing trickier and coupling less obvious. + # RoomStore, DirectoryStore, RegistrationStore, RoomMemberStore which is + # a bad assumption to make as it makes testing trickier and coupling + # less obvious. # get all rooms matching the room ID regex. room_entries = yield self.get_all_rooms() # RoomEntry list @@ -229,12 +230,34 @@ class ApplicationServiceStore(SQLBaseStore): r.room_id for r in room_alias_mappings if service.is_interested_in_alias(r.room_alias) ] - logging.debug(matching_alias_list) - logging.debug(matching_room_id_list) + room_ids_matching_alias_or_id = set( + matching_room_id_list + matching_alias_list + ) - # TODO get all rooms for every user for this AS. + # get all rooms for every user for this AS. This is scoped to users on + # this HS only. + user_list = yield self.get_all_users() + user_list = [ + u["name"] for u in user_list if + service.is_interested_in_user(u["name"]) + ] + rooms_for_user_matching_user_id = [] # RoomsForUser list + for user_id in user_list: + rooms_for_user = yield self.get_rooms_for_user(user_id) + rooms_for_user_matching_user_id += rooms_for_user + rooms_for_user_matching_user_id = set(rooms_for_user_matching_user_id) + + # make RoomsForUser tuples for room ids and aliases which are not in the + # main rooms_for_user_list - e.g. they are rooms which do not have AS + # registered users in it. + known_room_ids = [r.room_id for r in rooms_for_user_matching_user_id] + missing_rooms_for_user = [ + RoomsForUser(r, service.sender, "join") for r in + room_ids_matching_alias_or_id if r not in known_room_ids + ] + rooms_for_user_matching_user_id |= set(missing_rooms_for_user) - defer.returnValue([RoomsForUser("!foo:bar", service.sender, "join")]) + defer.returnValue(rooms_for_user_matching_user_id) @defer.inlineCallbacks def _populate_cache(self): diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 029b07cc66..7aff3dbd33 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -92,6 +92,13 @@ class RegistrationStore(SQLBaseStore): query, user_id ) + def get_all_users(self): + query = ("SELECT users.name FROM users") + return self._execute( + self.cursor_to_dict, + query + ) + def get_user_by_token(self, token): """Get a user from the given access token. -- cgit 1.4.1 From 59362454ddbcb3adf64d81da00dcd85cdb59a2e6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 26 Feb 2015 15:47:35 +0000 Subject: Must update pending_transactions map before yield'ing --- synapse/federation/transaction_queue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 8f1acbe590..741a4e7a1a 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -224,6 +224,8 @@ class TransactionQueue(object): ] try: + self.pending_transactions[destination] = 1 + limiter = yield get_retry_limiter( destination, self._clock, @@ -239,8 +241,6 @@ class TransactionQueue(object): len(pending_failures) ) - self.pending_transactions[destination] = 1 - logger.debug("TX [%s] Persisting transaction...", destination) transaction = Transaction.create_new( -- cgit 1.4.1 From 93d90765c4b09bc870fc91c6ddcf21fd4389659d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 26 Feb 2015 16:15:26 +0000 Subject: Initial implementation of federation server rate limiting --- synapse/federation/transport/__init__.py | 12 ++- synapse/federation/transport/server.py | 175 ++++++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transport/__init__.py b/synapse/federation/transport/__init__.py index 6800ac46c5..7028ca6947 100644 --- a/synapse/federation/transport/__init__.py +++ b/synapse/federation/transport/__init__.py @@ -21,7 +21,7 @@ support HTTPS), however individual pairings of servers may decide to communicate over a different (albeit still reliable) protocol. """ -from .server import TransportLayerServer +from .server import TransportLayerServer, FederationRateLimiter from .client import TransportLayerClient @@ -55,8 +55,18 @@ class TransportLayer(TransportLayerServer, TransportLayerClient): send requests """ self.keyring = homeserver.get_keyring() + self.clock = homeserver.get_clock() self.server_name = server_name self.server = server self.client = client self.request_handler = None self.received_handler = None + + self.ratelimiter = FederationRateLimiter( + self.clock, + window_size=10000, + sleep_limit=10, + sleep_msec=500, + reject_limit=50, + concurrent_requests=3, + ) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 2ffb37aa18..a9e625f127 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -16,9 +16,11 @@ from twisted.internet import defer from synapse.api.urls import FEDERATION_PREFIX as PREFIX -from synapse.api.errors import Codes, SynapseError +from synapse.api.errors import Codes, SynapseError, LimitExceededError +from synapse.util.async import sleep from synapse.util.logutils import log_function +import collections import logging import simplejson as json import re @@ -27,6 +29,163 @@ import re logger = logging.getLogger(__name__) +class FederationRateLimiter(object): + def __init__(self, clock, window_size, sleep_limit, sleep_msec, + reject_limit, concurrent_requests): + self.clock = clock + + self.window_size = window_size + self.sleep_limit = sleep_limit + self.sleep_msec = sleep_msec + self.reject_limit = reject_limit + self.concurrent_requests = concurrent_requests + + self.ratelimiters = {} + + def ratelimit(self, host): + return self.ratelimiters.setdefault( + host, + PerHostRatelimiter( + clock=self.clock, + window_size=self.window_size, + sleep_limit=self.sleep_limit, + sleep_msec=self.sleep_msec, + reject_limit=self.reject_limit, + concurrent_requests=self.concurrent_requests, + ) + ).ratelimit() + + +class PerHostRatelimiter(object): + def __init__(self, clock, window_size, sleep_limit, sleep_msec, + reject_limit, concurrent_requests): + self.clock = clock + + self.window_size = window_size + self.sleep_limit = sleep_limit + self.sleep_msec = sleep_msec + self.reject_limit = reject_limit + self.concurrent_requests = concurrent_requests + + self.sleeping_requests = set() + self.ready_request_queue = collections.OrderedDict() + self.current_processing = set() + self.request_times = [] + + def is_empty(self): + time_now = self.clock.time_msec() + self.request_times[:] = [ + r for r in self.request_times + if time_now - r < self.window_size + ] + + return not ( + self.ready_request_queue + or self.sleeping_requests + or self.current_processing + or self.request_times + ) + + def ratelimit(self): + request_id = object() + + def on_enter(): + return self._on_enter(request_id) + + def on_exit(exc_type, exc_val, exc_tb): + return self._on_exit(request_id) + + return ContextManagerFunction(on_enter, on_exit) + + def _on_enter(self, request_id): + time_now = self.clock.time_msec() + self.request_times[:] = [ + r for r in self.request_times + if time_now - r < self.window_size + ] + + queue_size = len(self.ready_request_queue) + len(self.sleeping_requests) + if queue_size > self.reject_limit: + raise LimitExceededError( + retry_after_ms=int( + self.window_size / self.sleep_limit + ), + ) + + self.request_times.append(time_now) + + def queue_request(): + if len(self.current_processing) > self.concurrent_requests: + logger.debug("Ratelimit [%s]: Queue req", id(request_id)) + queue_defer = defer.Deferred() + self.ready_request_queue[request_id] = queue_defer + return queue_defer + else: + return defer.succeed(None) + + logger.debug("Ratelimit [%s]: len(self.request_times)=%d", id(request_id), len(self.request_times)) + logger.debug("Ratelimit [%s]: len(self.request_times)=%d", id(request_id), len(self.request_times)) + + if len(self.request_times) > self.sleep_limit: + logger.debug("Ratelimit [%s]: sleeping req", id(request_id)) + ret_defer = sleep(self.sleep_msec/1000.0) + + self.sleeping_requests.add(request_id) + + def on_wait_finished(_): + logger.debug("Ratelimit [%s]: Finished sleeping", id(request_id)) + self.sleeping_requests.discard(request_id) + queue_defer = queue_request() + return queue_defer + + ret_defer.addBoth(on_wait_finished) + else: + ret_defer = queue_request() + + def on_start(r): + logger.debug("Ratelimit [%s]: Processing req", id(request_id)) + self.current_processing.add(request_id) + return r + + def on_err(r): + self.current_processing.discard(request_id) + return r + + def on_both(r): + # Ensure that we've properly cleaned up. + self.sleeping_requests.discard(request_id) + self.ready_request_queue.pop(request_id, None) + return r + + ret_defer.addCallbacks(on_start, on_err) + ret_defer.addBoth(on_both) + return ret_defer + + def _on_exit(self, request_id): + logger.debug("Ratelimit [%s]: Processed req", id(request_id)) + self.current_processing.discard(request_id) + try: + request_id, deferred = self.ready_request_queue.popitem() + self.current_processing.add(request_id) + deferred.callback(None) + except KeyError: + pass + + +class ContextManagerFunction(object): + def __init__(self, on_enter, on_exit): + self.on_enter = on_enter + self.on_exit = on_exit + + def __enter__(self): + if self.on_enter: + return self.on_enter() + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.on_exit: + return self.on_exit(exc_type, exc_val, exc_tb) + + class TransportLayerServer(object): """Handles incoming federation HTTP requests""" @@ -98,15 +257,23 @@ class TransportLayerServer(object): def new_handler(request, *args, **kwargs): try: (origin, content) = yield self._authenticate_request(request) - response = yield handler( - origin, content, request.args, *args, **kwargs - ) + with self.ratelimiter.ratelimit(origin) as d: + yield d + response = yield handler( + origin, content, request.args, *args, **kwargs + ) except: logger.exception("_authenticate_request failed") raise defer.returnValue(response) return new_handler + def rate_limit_origin(self, handler): + def new_handler(origin, *args, **kwargs): + response = yield handler(origin, *args, **kwargs) + defer.returnValue(response) + return new_handler() + @log_function def register_received_handler(self, handler): """ Register a handler that will be fired when we receive data. -- cgit 1.4.1 From dcec7175dc30754603618351b59bc72ff41d305b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 26 Feb 2015 16:23:01 +0000 Subject: Finish impl to get new events for AS. ASes should now be able to poll /events --- synapse/handlers/room.py | 4 ++- synapse/storage/stream.py | 62 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a8b0c95636..80f7ee3f12 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -566,7 +566,9 @@ class RoomEventSource(object): to_key = yield self.get_current_key() - app_service = self.store.get_app_service_by_user_id(user.to_string()) + app_service = yield self.store.get_app_service_by_user_id( + user.to_string() + ) if app_service: events, end_key = yield self.store.get_appservice_room_stream( service=app_service, diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3c8f3320f1..6946e9fe70 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -128,25 +128,73 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): class StreamStore(SQLBaseStore): + @defer.inlineCallbacks def get_appservice_room_stream(self, service, from_key, to_key, limit=0): # NB this lives here instead of appservice.py so we can reuse the # 'private' StreamToken class in this file. - logger.info("get_appservice_room_stream -> %s", service) - if limit: limit = max(limit, MAX_STREAM_SIZE) else: limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. - # from_id = _StreamToken.parse_stream_token(from_key) - # to_id = _StreamToken.parse_stream_token(to_key) + from_id = _StreamToken.parse_stream_token(from_key) + to_id = _StreamToken.parse_stream_token(to_key) if from_key == to_key: - return defer.succeed(([], to_key)) + defer.returnValue(([], to_key)) + return + + # Logic: + # - We want ALL events which match the AS room_id regex + # - We want ALL events which match the rooms represented by the AS + # room_alias regex + # - We want ALL events for rooms that AS users have joined. + # This is currently supported via get_app_service_rooms (which is used + # for the Notifier listener rooms). We can't reasonably make a SQL + # query for these room IDs, so we'll pull all the events between from/to + # and filter in python. + rooms_for_as = yield self.get_app_service_rooms(service) + room_ids_for_as = [r.room_id for r in rooms_for_as] + + # select all the events between from/to with a sensible limit + sql = ( + "SELECT e.event_id, e.room_id, e.stream_ordering FROM events AS e " + "WHERE e.stream_ordering > ? AND e.stream_ordering <= ? " + "ORDER BY stream_ordering ASC LIMIT %(limit)d " + ) % { + "limit": limit + } + + def f(txn): + txn.execute(sql, (from_id.stream, to_id.stream,)) + + rows = self.cursor_to_dict(txn) + + ret = self._get_events_txn( + txn, + # apply the filter on the room id list + [ + r["event_id"] for r in rows + if r["room_id"] in room_ids_for_as + ], + get_prev_content=True + ) + + self._set_before_and_after(ret, rows) + + if rows: + key = "s%d" % max([r["stream_ordering"] for r in rows]) + + else: + # Assume we didn't get anything because there was nothing to + # get. + key = to_key + + return ret, key - # TODO stub - return defer.succeed(([], to_key)) + results = yield self.runInteraction("get_appservice_room_stream", f) + defer.returnValue(results) @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, -- cgit 1.4.1 From f0995436e7951448d8be3c372f4002845a111a7d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 26 Feb 2015 17:21:17 +0000 Subject: Check for membership invite events correctly. --- synapse/storage/stream.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 6946e9fe70..5d01ecf200 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -36,12 +36,14 @@ what sort order was used: from twisted.internet import defer from ._base import SQLBaseStore +from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.util.logutils import log_function from collections import namedtuple import logging +import simplejson logger = logging.getLogger(__name__) @@ -159,13 +161,30 @@ class StreamStore(SQLBaseStore): # select all the events between from/to with a sensible limit sql = ( - "SELECT e.event_id, e.room_id, e.stream_ordering FROM events AS e " + "SELECT e.event_id, e.room_id, e.type, e.unrecognized_keys, " + "e.stream_ordering FROM events AS e " "WHERE e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { "limit": limit } + + def app_service_interested(row): + if row["room_id"] in room_ids_for_as: + return True + + if row["type"] == EventTypes.Member: + # load up the content to inspect if some user the AS is + # interested in was invited to a room. We'll be passing this + # through _get_events_txn later, so ignore the fact that this + # may be a redacted event. + event_content = simplejson.loads(row["unrecognized_keys"]) + if (service.is_interested_in_user( + event_content.get("state_key"))): + return True + return False + def f(txn): txn.execute(sql, (from_id.stream, to_id.stream,)) @@ -176,7 +195,7 @@ class StreamStore(SQLBaseStore): # apply the filter on the room id list [ r["event_id"] for r in rows - if r["room_id"] in room_ids_for_as + if app_service_interested(r) ], get_prev_content=True ) -- cgit 1.4.1 From 19590881568f5aafd1b1d0b12cd6c10954ee60b1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 26 Feb 2015 18:07:44 +0000 Subject: Add API for getting/setting enabled-ness of push rules. --- synapse/push/baserules.py | 8 +++--- synapse/rest/client/v1/push_rule.py | 54 +++++++++++++++++++++++++++++++++---- synapse/storage/push_rule.py | 24 +++++++++++++++++ 3 files changed, 77 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index ba9a181b56..f4d2be11f6 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -37,7 +37,7 @@ def make_base_rules(user, kind): for r in rules: r['priority_class'] = PRIORITY_CLASS_MAP[kind] - r['default'] = True + r['default'] = True # Deprecated, left for backwards compat return rules @@ -45,7 +45,7 @@ def make_base_rules(user, kind): def make_base_content_rules(user): return [ { - 'rule_id': '.m.rule.contains_user_name', + 'rule_id': 'global/content/.m.rule.contains_user_name', 'conditions': [ { 'kind': 'event_match', @@ -67,7 +67,7 @@ def make_base_content_rules(user): def make_base_override_rules(): return [ { - 'rule_id': '.m.rule.contains_display_name', + 'rule_id': 'global/override/.m.rule.contains_display_name', 'conditions': [ { 'kind': 'contains_display_name' @@ -82,7 +82,7 @@ def make_base_override_rules(): ] }, { - 'rule_id': '.m.rule.room_two_members', + 'rule_id': 'global/override/.m.rule.room_two_members', 'conditions': [ { 'kind': 'room_member_count', diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 73ba0494e6..c6133a8688 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -50,6 +50,10 @@ class PushRuleRestServlet(ClientV1RestServlet): content = _parse_json(request) + if 'attr' in spec: + self.set_rule_attr(user.to_string(), spec, content) + defer.returnValue((200, {})) + try: (conditions, actions) = _rule_tuple_from_request_object( spec['template'], @@ -124,6 +128,9 @@ class PushRuleRestServlet(ClientV1RestServlet): rules['global'] = _add_empty_priority_class_arrays(rules['global']) + enabled_map = yield self.hs.get_datastore().\ + get_push_rules_enabled_for_user_name(user.to_string()) + for r in ruleslist: rulearray = None @@ -149,6 +156,9 @@ class PushRuleRestServlet(ClientV1RestServlet): template_rule = _rule_to_template(r) if template_rule: + template_rule['enabled'] = True + if r['rule_id'] in enabled_map: + template_rule['enabled'] = enabled_map[r['rule_id']] rulearray.append(template_rule) path = request.postpath[1:] @@ -189,6 +199,24 @@ class PushRuleRestServlet(ClientV1RestServlet): def on_OPTIONS(self, _): return 200, {} + def set_rule_attr(self, user_name, spec, val): + if spec['attr'] == 'enabled': + if not isinstance(val, bool): + raise SynapseError(400, "Value for 'enabled' must be boolean") + namespaced_rule_id = _namespaced_rule_id_from_spec(spec) + self.hs.get_datastore().set_push_rule_enabled( + user_name, namespaced_rule_id, val + ) + else: + raise UnrecognizedRequestError() + + def get_rule_attr(self, user_name, namespaced_rule_id, attr): + if attr == 'enabled': + return self.hs.get_datastore().get_push_rule_enabled_by_user_name_rule_id( + user_name, namespaced_rule_id + ) + else: + raise UnrecognizedRequestError() def _rule_spec_from_path(path): if len(path) < 2: @@ -226,6 +254,12 @@ def _rule_spec_from_path(path): } if device: spec['profile_tag'] = device + + path = path[1:] + + if len(path) > 0 and len(path[0]) > 0: + spec['attr'] = path[0] + return spec @@ -319,10 +353,23 @@ def _filter_ruleset_with_path(ruleset, path): if path[0] == '': return ruleset[template_kind] rule_id = path[0] + + the_rule = None for r in ruleset[template_kind]: if r['rule_id'] == rule_id: - return r - raise NotFoundError + the_rule = r + if the_rule is None: + raise NotFoundError + + path = path[1:] + if len(path) == 0: + return the_rule + + attr = path[0] + if attr in the_rule: + return the_rule[attr] + else: + raise UnrecognizedRequestError() def _priority_class_from_spec(spec): @@ -399,9 +446,6 @@ class InvalidRuleException(Exception): def _parse_json(request): try: content = json.loads(request.content.read()) - if type(content) != dict: - raise SynapseError(400, "Content must be a JSON object.", - errcode=Codes.NOT_JSON) return content except ValueError: raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index cd8d0f6dde..6c38565773 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -56,6 +56,17 @@ class PushRuleStore(SQLBaseStore): {r['rule_id']: False if r['enabled'] == 0 else True for r in results} ) + @defer.inlineCallbacks + def get_push_rule_enabled_by_user_name_rule_id(self, user_name, rule_id): + results = yield self._simple_select_list( + PushRuleEnableTable.table_name, + {'user_name': user_name, 'rule_id': rule_id}, + ['enabled'] + ) + if len(results) == 0: + defer.returnValue(True) + defer.returnValue(results[0]) + @defer.inlineCallbacks def add_push_rule(self, before, after, **kwargs): vals = copy.copy(kwargs) @@ -204,6 +215,19 @@ class PushRuleStore(SQLBaseStore): {'user_name': user_name, 'rule_id': rule_id} ) + @defer.inlineCallbacks + def set_push_rule_enabled(self, user_name, rule_id, enabled): + if enabled: + yield self._simple_delete_one( + PushRuleEnableTable.table_name, + {'user_name': user_name, 'rule_id': rule_id} + ) + else: + yield self._simple_upsert( + PushRuleEnableTable.table_name, + {'user_name': user_name, 'rule_id': rule_id}, + {'enabled': False} + ) class RuleNotFoundException(Exception): pass -- cgit 1.4.1 From cfac3b7873e95f1c643708b9a3fddc7c37896826 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 26 Feb 2015 18:58:14 +0000 Subject: SYN-267 Add a fallback rule as an explicit server default rule and make the default dont-notify so you effectively have a "notify for everything else" switch you can turn on and off. --- synapse/push/__init__.py | 2 +- synapse/push/baserules.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 40fae91ab5..d4da05f093 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -32,7 +32,7 @@ class Pusher(object): INITIAL_BACKOFF = 1000 MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - DEFAULT_ACTIONS = ['notify'] + DEFAULT_ACTIONS = ['dont-notify'] INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$") diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index f4d2be11f6..09878b63c1 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -32,6 +32,8 @@ def make_base_rules(user, kind): if kind == 'override': rules = make_base_override_rules() + elif kind == 'underride': + rules = make_base_underride_rules() elif kind == 'content': rules = make_base_content_rules(user) @@ -98,3 +100,16 @@ def make_base_override_rules(): ] } ] + + +def make_base_underride_rules(): + return [ + { + 'rule_id': 'global/underride/.m.rule.fallback', + 'conditions': [ + ], + 'actions': [ + 'notify', + ] + }, + ] -- cgit 1.4.1 From 1cc77145d483c4a6cea75fafbe83a1661b8b61cd Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 27 Feb 2015 09:39:12 +0000 Subject: Notify appservices of invites mid-poll. This requires the notifier to have knowledge of appservice listeners so it can do the regex checks on incoming invites to see if the state_key matches. It isn't enough to just rely on the room listeners and store.get_app_service_rooms as the room will initially not exist or won't be on the ASes radar due to having none of its users in the room. --- synapse/notifier.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/notifier.py b/synapse/notifier.py index 2475f3ffbe..09d23e79b8 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -36,8 +36,10 @@ class _NotificationListener(object): so that it can remove itself from the indexes in the Notifier class. """ - def __init__(self, user, rooms, from_token, limit, timeout, deferred): + def __init__(self, user, rooms, from_token, limit, timeout, deferred, + appservice=None): self.user = user + self.appservice = appservice self.from_token = from_token self.limit = limit self.timeout = timeout @@ -65,6 +67,10 @@ class _NotificationListener(object): lst.discard(self) notifier.user_to_listeners.get(self.user, set()).discard(self) + if self.appservice: + notifier.appservice_to_listeners.get( + self.appservice, set() + ).discard(self) class Notifier(object): @@ -79,6 +85,7 @@ class Notifier(object): self.rooms_to_listeners = {} self.user_to_listeners = {} + self.appservice_to_listeners = {} self.event_sources = hs.get_event_sources() @@ -114,6 +121,17 @@ class Notifier(object): for user in extra_users: listeners |= self.user_to_listeners.get(user, set()).copy() + for appservice in self.appservice_to_listeners: + # TODO (kegan): Redundant appservice listener checks? + # App services will already be in the rooms_to_listeners set, but + # that isn't enough. They need to be checked here in order to + # receive *invites* for users they are interested in. Does this + # make the rooms_to_listeners check somewhat obselete? + if appservice.is_interested(event): + listeners |= self.appservice_to_listeners.get( + appservice, set() + ).copy() + logger.debug("on_new_room_event listeners %s", listeners) # TODO (erikj): Can we make this more efficient by hitting the @@ -280,6 +298,10 @@ class Notifier(object): if not from_token: from_token = yield self.event_sources.get_current_token() + appservice = yield self.hs.get_datastore().get_app_service_by_user_id( + user.to_string() + ) + listener = _NotificationListener( user, rooms, @@ -287,6 +309,7 @@ class Notifier(object): limit, timeout, deferred, + appservice=appservice ) def _timeout_listener(): @@ -319,6 +342,11 @@ class Notifier(object): self.user_to_listeners.setdefault(listener.user, set()).add(listener) + if listener.appservice: + self.appservice_to_listeners.setdefault( + listener.appservice, set() + ).add(listener) + @defer.inlineCallbacks @log_function def _check_for_updates(self, listener): -- cgit 1.4.1 From 806a6c886aaa695a7dbfd35f71b9cc59941b8366 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 27 Feb 2015 09:48:57 +0000 Subject: PEP8 --- synapse/storage/stream.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 5d01ecf200..09417bd147 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -169,7 +169,6 @@ class StreamStore(SQLBaseStore): "limit": limit } - def app_service_interested(row): if row["room_id"] in room_ids_for_as: return True -- cgit 1.4.1 From 16b90764adb8f2ab49b1853855d0fb739b79d245 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 27 Feb 2015 10:44:32 +0000 Subject: Convert expected format for AS regex to include exclusivity. Previously you just specified the regex as a string, now it expects a JSON object with a 'regex' key and an 'exclusive' boolean, as per spec. --- synapse/appservice/__init__.py | 26 ++++++++++++++++++------- synapse/rest/appservice/v1/register.py | 35 ++++++---------------------------- synapse/storage/appservice.py | 16 +++++++++++++--- 3 files changed, 38 insertions(+), 39 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 381b4cfc4a..b5e7ac16ba 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -46,19 +46,31 @@ class ApplicationService(object): def _check_namespaces(self, namespaces): # Sanity check that it is of the form: # { - # users: ["regex",...], - # aliases: ["regex",...], - # rooms: ["regex",...], + # users: [ {regex: "[A-z]+.*", exclusive: true}, ...], + # aliases: [ {regex: "[A-z]+.*", exclusive: true}, ...], + # rooms: [ {regex: "[A-z]+.*", exclusive: true}, ...], # } if not namespaces: return None for ns in ApplicationService.NS_LIST: + if ns not in namespaces: + namespaces[ns] = [] + continue + if type(namespaces[ns]) != list: - raise ValueError("Bad namespace value for '%s'", ns) - for regex in namespaces[ns]: - if not isinstance(regex, basestring): - raise ValueError("Expected string regex for ns '%s'", ns) + raise ValueError("Bad namespace value for '%s'" % ns) + for regex_obj in namespaces[ns]: + if not isinstance(regex_obj, dict): + raise ValueError("Expected dict regex for ns '%s'" % ns) + if not isinstance(regex_obj.get("exclusive"), bool): + raise ValueError( + "Expected bool for 'exclusive' in ns '%s'" % ns + ) + if not isinstance(regex_obj.get("regex"), basestring): + raise ValueError( + "Expected string for 'regex' in ns '%s'" % ns + ) return namespaces def _matches_regex(self, test_string, namespace_key): diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 3bd0c1220c..a4f6159773 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -48,18 +48,12 @@ class RegisterRestServlet(AppServiceRestServlet): 400, "Missed required keys: as_token(str) / url(str)." ) - namespaces = { - "users": [], - "rooms": [], - "aliases": [] - } - - if "namespaces" in params: - self._parse_namespace(namespaces, params["namespaces"], "users") - self._parse_namespace(namespaces, params["namespaces"], "rooms") - self._parse_namespace(namespaces, params["namespaces"], "aliases") - - app_service = ApplicationService(as_token, as_url, namespaces) + try: + app_service = ApplicationService( + as_token, as_url, params["namespaces"] + ) + except ValueError as e: + raise SynapseError(400, e.message) app_service = yield self.handler.register(app_service) hs_token = app_service.hs_token @@ -68,23 +62,6 @@ class RegisterRestServlet(AppServiceRestServlet): "hs_token": hs_token })) - def _parse_namespace(self, target_ns, origin_ns, ns): - if ns not in target_ns or ns not in origin_ns: - return # nothing to parse / map through to. - - possible_regex_list = origin_ns[ns] - if not type(possible_regex_list) == list: - raise SynapseError(400, "Namespace %s isn't an array." % ns) - - for regex in possible_regex_list: - if not isinstance(regex, basestring): - raise SynapseError( - 400, "Regex '%s' isn't a string in namespace %s" % - (regex, ns) - ) - - target_ns[ns] = origin_ns[ns] - class UnregisterRestServlet(AppServiceRestServlet): """Handles AS registration with the home server. diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index dc3666efd4..a3aa41e5fc 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import simplejson +from simplejson import JSONDecodeError from twisted.internet import defer from synapse.api.errors import StoreError @@ -23,12 +25,18 @@ from ._base import SQLBaseStore logger = logging.getLogger(__name__) +def log_failure(failure): + logger.error("Failed to detect application services: %s", failure.value) + logger.error(failure.getTraceback()) + + class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.services_cache = [] self.cache_defer = self._populate_cache() + self.cache_defer.addErrback(log_failure) @defer.inlineCallbacks def unregister_app_service(self, token): @@ -128,11 +136,11 @@ class ApplicationServiceStore(SQLBaseStore): ) for (ns_int, ns_str) in enumerate(ApplicationService.NS_LIST): if ns_str in service.namespaces: - for regex in service.namespaces[ns_str]: + for regex_obj in service.namespaces[ns_str]: txn.execute( "INSERT INTO application_services_regex(" "as_id, namespace, regex) values(?,?,?)", - (as_id, ns_int, regex) + (as_id, ns_int, simplejson.dumps(regex_obj)) ) return True @@ -215,10 +223,12 @@ class ApplicationServiceStore(SQLBaseStore): try: services[as_token]["namespaces"][ ApplicationService.NS_LIST[ns_int]].append( - res["regex"] + simplejson.loads(res["regex"]) ) except IndexError: logger.error("Bad namespace enum '%s'. %s", ns_int, res) + except JSONDecodeError: + logger.error("Bad regex object '%s'", res["regex"]) # TODO get last successful txn id f.e. service for service in services.values(): -- cgit 1.4.1 From 40c9896705208b1455192b496e3e8e3bc9e9c0a9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 27 Feb 2015 11:03:56 +0000 Subject: Add functions to return whether an AS has exclusively claimed a matching namespace. --- synapse/appservice/__init__.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index b5e7ac16ba..a268a6bcc4 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -73,7 +73,7 @@ class ApplicationService(object): ) return namespaces - def _matches_regex(self, test_string, namespace_key): + def _matches_regex(self, test_string, namespace_key, return_obj=False): if not isinstance(test_string, basestring): logger.error( "Expected a string to test regex against, but got %s", @@ -81,11 +81,19 @@ class ApplicationService(object): ) return False - for regex in self.namespaces[namespace_key]: - if re.match(regex, test_string): + for regex_obj in self.namespaces[namespace_key]: + if re.match(regex_obj["regex"], test_string): + if return_obj: + return regex_obj return True return False + def _is_exclusive(self, ns_key, test_string): + regex_obj = self._matches_regex(test_string, ns_key, return_obj=True) + if regex_obj: + return regex_obj["exclusive"] + return False + def _matches_user(self, event, member_list): if (hasattr(event, "sender") and self.is_interested_in_user(event.sender)): @@ -155,5 +163,14 @@ class ApplicationService(object): def is_interested_in_room(self, room_id): return self._matches_regex(room_id, ApplicationService.NS_ROOMS) + def is_exclusive_user(self, user_id): + return self._is_exclusive(ApplicationService.NS_USERS, user_id) + + def is_exclusive_alias(self, alias): + return self._is_exclusive(ApplicationService.NS_ALIASES, alias) + + def is_exclusive_room(self, room_id): + return self._is_exclusive(ApplicationService.NS_ROOMS, room_id) + def __str__(self): return "ApplicationService: %s" % (self.__dict__,) -- cgit 1.4.1 From 58ff0660642e6ef8f9fe5672fb50e5d75a569479 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 27 Feb 2015 13:51:41 +0000 Subject: Implement exclusive namespace checks. --- synapse/handlers/directory.py | 14 ++++++++++++-- synapse/handlers/register.py | 11 ++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 11d20a5d2d..f76febee8f 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -232,13 +232,23 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def can_modify_alias(self, alias, user_id=None): + # Any application service "interested" in an alias they are regexing on + # can modify the alias. + # Users can only modify the alias if ALL the interested services have + # non-exclusive locks on the alias (or there are no interested services) services = yield self.store.get_app_services() interested_services = [ s for s in services if s.is_interested_in_alias(alias.to_string()) ] + for service in interested_services: if user_id == service.sender: - # this user IS the app service + # this user IS the app service so they can do whatever they like defer.returnValue(True) return - defer.returnValue(len(interested_services) == 0) + elif service.is_exclusive_alias(alias.to_string()): + # another service has an exclusive lock on this alias. + defer.returnValue(False) + return + # either no interested services, or no service with an exclusive lock + defer.returnValue(True) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 516a936cee..cda4a8502a 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -201,11 +201,12 @@ class RegistrationHandler(BaseHandler): interested_services = [ s for s in services if s.is_interested_in_user(user_id) ] - if len(interested_services) > 0: - raise SynapseError( - 400, "This user ID is reserved by an application service.", - errcode=Codes.EXCLUSIVE - ) + for service in interested_services: + if service.is_exclusive_user(user_id): + raise SynapseError( + 400, "This user ID is reserved by an application service.", + errcode=Codes.EXCLUSIVE + ) def _generate_token(self, user_id): # urlsafe variant uses _ and - so use . as the separator and replace -- cgit 1.4.1 From 9dc9118e552bfddaa7579b4ded8b1a0da7eff0e6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 27 Feb 2015 15:16:47 +0000 Subject: Document FederationRateLimiter --- synapse/federation/transport/server.py | 59 +++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index a9e625f127..390e54b9fb 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -32,6 +32,21 @@ logger = logging.getLogger(__name__) class FederationRateLimiter(object): def __init__(self, clock, window_size, sleep_limit, sleep_msec, reject_limit, concurrent_requests): + """ + Args: + clock (Clock) + window_size (int): The window size in milliseconds. + sleep_limit (int): The number of requests received in the last + `window_size` milliseconds before we artificially start + delaying processing of requests. + sleep_msec (int): The number of milliseconds to delay processing + of incoming requests by. + reject_limit (int): The maximum number of requests that are can be + queued for processing before we start rejecting requests with + a 429 Too Many Requests response. + concurrent_requests (int): The number of concurrent requests to + process. + """ self.clock = clock self.window_size = window_size @@ -43,9 +58,23 @@ class FederationRateLimiter(object): self.ratelimiters = {} def ratelimit(self, host): + """Used to ratelimit an incoming request from given host + + Example usage: + + with rate_limiter.ratelimit(origin) as wait_deferred: + yield wait_deferred + # Handle request ... + + Args: + host (str): Origin of incoming request. + + Returns: + _PerHostRatelimiter + """ return self.ratelimiters.setdefault( host, - PerHostRatelimiter( + _PerHostRatelimiter( clock=self.clock, window_size=self.window_size, sleep_limit=self.sleep_limit, @@ -56,7 +85,7 @@ class FederationRateLimiter(object): ).ratelimit() -class PerHostRatelimiter(object): +class _PerHostRatelimiter(object): def __init__(self, clock, window_size, sleep_limit, sleep_msec, reject_limit, concurrent_requests): self.clock = clock @@ -123,17 +152,25 @@ class PerHostRatelimiter(object): else: return defer.succeed(None) - logger.debug("Ratelimit [%s]: len(self.request_times)=%d", id(request_id), len(self.request_times)) - logger.debug("Ratelimit [%s]: len(self.request_times)=%d", id(request_id), len(self.request_times)) + logger.debug( + "Ratelimit [%s]: len(self.request_times)=%d", + id(request_id), len(self.request_times), + ) if len(self.request_times) > self.sleep_limit: - logger.debug("Ratelimit [%s]: sleeping req", id(request_id)) + logger.debug( + "Ratelimit [%s]: sleeping req", + id(request_id), + ) ret_defer = sleep(self.sleep_msec/1000.0) self.sleeping_requests.add(request_id) def on_wait_finished(_): - logger.debug("Ratelimit [%s]: Finished sleeping", id(request_id)) + logger.debug( + "Ratelimit [%s]: Finished sleeping", + id(request_id), + ) self.sleeping_requests.discard(request_id) queue_defer = queue_request() return queue_defer @@ -143,7 +180,10 @@ class PerHostRatelimiter(object): ret_defer = queue_request() def on_start(r): - logger.debug("Ratelimit [%s]: Processing req", id(request_id)) + logger.debug( + "Ratelimit [%s]: Processing req", + id(request_id), + ) self.current_processing.add(request_id) return r @@ -162,7 +202,10 @@ class PerHostRatelimiter(object): return ret_defer def _on_exit(self, request_id): - logger.debug("Ratelimit [%s]: Processed req", id(request_id)) + logger.debug( + "Ratelimit [%s]: Processed req", + id(request_id), + ) self.current_processing.discard(request_id) try: request_id, deferred = self.ready_request_queue.popitem() -- cgit 1.4.1 From 0554d0708225afe13d141bd00e3aaca2509f3f78 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 27 Feb 2015 15:41:52 +0000 Subject: Move federation rate limiting out of transport layer --- synapse/federation/transport/__init__.py | 4 +- synapse/federation/transport/server.py | 204 +--------------------------- synapse/util/ratelimitutils.py | 226 +++++++++++++++++++++++++++++++ 3 files changed, 230 insertions(+), 204 deletions(-) create mode 100644 synapse/util/ratelimitutils.py (limited to 'synapse') diff --git a/synapse/federation/transport/__init__.py b/synapse/federation/transport/__init__.py index 7028ca6947..f0283b5105 100644 --- a/synapse/federation/transport/__init__.py +++ b/synapse/federation/transport/__init__.py @@ -21,9 +21,11 @@ support HTTPS), however individual pairings of servers may decide to communicate over a different (albeit still reliable) protocol. """ -from .server import TransportLayerServer, FederationRateLimiter +from .server import TransportLayerServer from .client import TransportLayerClient +from synapse.util.ratelimitutils import FederationRateLimiter + class TransportLayer(TransportLayerServer, TransportLayerClient): """This is a basic implementation of the transport layer that translates diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 390e54b9fb..fce9c0195e 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -16,11 +16,9 @@ from twisted.internet import defer from synapse.api.urls import FEDERATION_PREFIX as PREFIX -from synapse.api.errors import Codes, SynapseError, LimitExceededError -from synapse.util.async import sleep +from synapse.api.errors import Codes, SynapseError from synapse.util.logutils import log_function -import collections import logging import simplejson as json import re @@ -29,206 +27,6 @@ import re logger = logging.getLogger(__name__) -class FederationRateLimiter(object): - def __init__(self, clock, window_size, sleep_limit, sleep_msec, - reject_limit, concurrent_requests): - """ - Args: - clock (Clock) - window_size (int): The window size in milliseconds. - sleep_limit (int): The number of requests received in the last - `window_size` milliseconds before we artificially start - delaying processing of requests. - sleep_msec (int): The number of milliseconds to delay processing - of incoming requests by. - reject_limit (int): The maximum number of requests that are can be - queued for processing before we start rejecting requests with - a 429 Too Many Requests response. - concurrent_requests (int): The number of concurrent requests to - process. - """ - self.clock = clock - - self.window_size = window_size - self.sleep_limit = sleep_limit - self.sleep_msec = sleep_msec - self.reject_limit = reject_limit - self.concurrent_requests = concurrent_requests - - self.ratelimiters = {} - - def ratelimit(self, host): - """Used to ratelimit an incoming request from given host - - Example usage: - - with rate_limiter.ratelimit(origin) as wait_deferred: - yield wait_deferred - # Handle request ... - - Args: - host (str): Origin of incoming request. - - Returns: - _PerHostRatelimiter - """ - return self.ratelimiters.setdefault( - host, - _PerHostRatelimiter( - clock=self.clock, - window_size=self.window_size, - sleep_limit=self.sleep_limit, - sleep_msec=self.sleep_msec, - reject_limit=self.reject_limit, - concurrent_requests=self.concurrent_requests, - ) - ).ratelimit() - - -class _PerHostRatelimiter(object): - def __init__(self, clock, window_size, sleep_limit, sleep_msec, - reject_limit, concurrent_requests): - self.clock = clock - - self.window_size = window_size - self.sleep_limit = sleep_limit - self.sleep_msec = sleep_msec - self.reject_limit = reject_limit - self.concurrent_requests = concurrent_requests - - self.sleeping_requests = set() - self.ready_request_queue = collections.OrderedDict() - self.current_processing = set() - self.request_times = [] - - def is_empty(self): - time_now = self.clock.time_msec() - self.request_times[:] = [ - r for r in self.request_times - if time_now - r < self.window_size - ] - - return not ( - self.ready_request_queue - or self.sleeping_requests - or self.current_processing - or self.request_times - ) - - def ratelimit(self): - request_id = object() - - def on_enter(): - return self._on_enter(request_id) - - def on_exit(exc_type, exc_val, exc_tb): - return self._on_exit(request_id) - - return ContextManagerFunction(on_enter, on_exit) - - def _on_enter(self, request_id): - time_now = self.clock.time_msec() - self.request_times[:] = [ - r for r in self.request_times - if time_now - r < self.window_size - ] - - queue_size = len(self.ready_request_queue) + len(self.sleeping_requests) - if queue_size > self.reject_limit: - raise LimitExceededError( - retry_after_ms=int( - self.window_size / self.sleep_limit - ), - ) - - self.request_times.append(time_now) - - def queue_request(): - if len(self.current_processing) > self.concurrent_requests: - logger.debug("Ratelimit [%s]: Queue req", id(request_id)) - queue_defer = defer.Deferred() - self.ready_request_queue[request_id] = queue_defer - return queue_defer - else: - return defer.succeed(None) - - logger.debug( - "Ratelimit [%s]: len(self.request_times)=%d", - id(request_id), len(self.request_times), - ) - - if len(self.request_times) > self.sleep_limit: - logger.debug( - "Ratelimit [%s]: sleeping req", - id(request_id), - ) - ret_defer = sleep(self.sleep_msec/1000.0) - - self.sleeping_requests.add(request_id) - - def on_wait_finished(_): - logger.debug( - "Ratelimit [%s]: Finished sleeping", - id(request_id), - ) - self.sleeping_requests.discard(request_id) - queue_defer = queue_request() - return queue_defer - - ret_defer.addBoth(on_wait_finished) - else: - ret_defer = queue_request() - - def on_start(r): - logger.debug( - "Ratelimit [%s]: Processing req", - id(request_id), - ) - self.current_processing.add(request_id) - return r - - def on_err(r): - self.current_processing.discard(request_id) - return r - - def on_both(r): - # Ensure that we've properly cleaned up. - self.sleeping_requests.discard(request_id) - self.ready_request_queue.pop(request_id, None) - return r - - ret_defer.addCallbacks(on_start, on_err) - ret_defer.addBoth(on_both) - return ret_defer - - def _on_exit(self, request_id): - logger.debug( - "Ratelimit [%s]: Processed req", - id(request_id), - ) - self.current_processing.discard(request_id) - try: - request_id, deferred = self.ready_request_queue.popitem() - self.current_processing.add(request_id) - deferred.callback(None) - except KeyError: - pass - - -class ContextManagerFunction(object): - def __init__(self, on_enter, on_exit): - self.on_enter = on_enter - self.on_exit = on_exit - - def __enter__(self): - if self.on_enter: - return self.on_enter() - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.on_exit: - return self.on_exit(exc_type, exc_val, exc_tb) - - class TransportLayerServer(object): """Handles incoming federation HTTP requests""" diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py new file mode 100644 index 0000000000..259d5f6f88 --- /dev/null +++ b/synapse/util/ratelimitutils.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import LimitExceededError + +from synapse.util.async import sleep + +import collections +import logging + + +logger = logging.getLogger(__name__) + + +class FederationRateLimiter(object): + def __init__(self, clock, window_size, sleep_limit, sleep_msec, + reject_limit, concurrent_requests): + """ + Args: + clock (Clock) + window_size (int): The window size in milliseconds. + sleep_limit (int): The number of requests received in the last + `window_size` milliseconds before we artificially start + delaying processing of requests. + sleep_msec (int): The number of milliseconds to delay processing + of incoming requests by. + reject_limit (int): The maximum number of requests that are can be + queued for processing before we start rejecting requests with + a 429 Too Many Requests response. + concurrent_requests (int): The number of concurrent requests to + process. + """ + self.clock = clock + + self.window_size = window_size + self.sleep_limit = sleep_limit + self.sleep_msec = sleep_msec + self.reject_limit = reject_limit + self.concurrent_requests = concurrent_requests + + self.ratelimiters = {} + + def ratelimit(self, host): + """Used to ratelimit an incoming request from given host + + Example usage: + + with rate_limiter.ratelimit(origin) as wait_deferred: + yield wait_deferred + # Handle request ... + + Args: + host (str): Origin of incoming request. + + Returns: + _PerHostRatelimiter + """ + return self.ratelimiters.setdefault( + host, + _PerHostRatelimiter( + clock=self.clock, + window_size=self.window_size, + sleep_limit=self.sleep_limit, + sleep_msec=self.sleep_msec, + reject_limit=self.reject_limit, + concurrent_requests=self.concurrent_requests, + ) + ).ratelimit() + + +class _PerHostRatelimiter(object): + def __init__(self, clock, window_size, sleep_limit, sleep_msec, + reject_limit, concurrent_requests): + self.clock = clock + + self.window_size = window_size + self.sleep_limit = sleep_limit + self.sleep_msec = sleep_msec + self.reject_limit = reject_limit + self.concurrent_requests = concurrent_requests + + self.sleeping_requests = set() + self.ready_request_queue = collections.OrderedDict() + self.current_processing = set() + self.request_times = [] + + def is_empty(self): + time_now = self.clock.time_msec() + self.request_times[:] = [ + r for r in self.request_times + if time_now - r < self.window_size + ] + + return not ( + self.ready_request_queue + or self.sleeping_requests + or self.current_processing + or self.request_times + ) + + def ratelimit(self): + request_id = object() + + def on_enter(): + return self._on_enter(request_id) + + def on_exit(exc_type, exc_val, exc_tb): + return self._on_exit(request_id) + + return ContextManagerFunction(on_enter, on_exit) + + def _on_enter(self, request_id): + time_now = self.clock.time_msec() + self.request_times[:] = [ + r for r in self.request_times + if time_now - r < self.window_size + ] + + queue_size = len(self.ready_request_queue) + len(self.sleeping_requests) + if queue_size > self.reject_limit: + raise LimitExceededError( + retry_after_ms=int( + self.window_size / self.sleep_limit + ), + ) + + self.request_times.append(time_now) + + def queue_request(): + if len(self.current_processing) > self.concurrent_requests: + logger.debug("Ratelimit [%s]: Queue req", id(request_id)) + queue_defer = defer.Deferred() + self.ready_request_queue[request_id] = queue_defer + return queue_defer + else: + return defer.succeed(None) + + logger.debug( + "Ratelimit [%s]: len(self.request_times)=%d", + id(request_id), len(self.request_times), + ) + + if len(self.request_times) > self.sleep_limit: + logger.debug( + "Ratelimit [%s]: sleeping req", + id(request_id), + ) + ret_defer = sleep(self.sleep_msec/1000.0) + + self.sleeping_requests.add(request_id) + + def on_wait_finished(_): + logger.debug( + "Ratelimit [%s]: Finished sleeping", + id(request_id), + ) + self.sleeping_requests.discard(request_id) + queue_defer = queue_request() + return queue_defer + + ret_defer.addBoth(on_wait_finished) + else: + ret_defer = queue_request() + + def on_start(r): + logger.debug( + "Ratelimit [%s]: Processing req", + id(request_id), + ) + self.current_processing.add(request_id) + return r + + def on_err(r): + self.current_processing.discard(request_id) + return r + + def on_both(r): + # Ensure that we've properly cleaned up. + self.sleeping_requests.discard(request_id) + self.ready_request_queue.pop(request_id, None) + return r + + ret_defer.addCallbacks(on_start, on_err) + ret_defer.addBoth(on_both) + return ret_defer + + def _on_exit(self, request_id): + logger.debug( + "Ratelimit [%s]: Processed req", + id(request_id), + ) + self.current_processing.discard(request_id) + try: + request_id, deferred = self.ready_request_queue.popitem() + self.current_processing.add(request_id) + deferred.callback(None) + except KeyError: + pass + + +class ContextManagerFunction(object): + def __init__(self, on_enter, on_exit): + self.on_enter = on_enter + self.on_exit = on_exit + + def __enter__(self): + if self.on_enter: + return self.on_enter() + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.on_exit: + return self.on_exit(exc_type, exc_val, exc_tb) -- cgit 1.4.1 From 2a6dedd7cc349201e8ad1607835221ada1c0828f Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 27 Feb 2015 18:38:56 +0000 Subject: It's set_tweak now, not set_sound --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index c6133a8688..5bfdb29907 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -309,7 +309,7 @@ def _rule_tuple_from_request_object(rule_template, rule_id, req_obj, device=None for a in actions: if a in ['notify', 'dont_notify', 'coalesce']: pass - elif isinstance(a, dict) and 'set_sound' in a: + elif isinstance(a, dict) and 'set_tweak' in a: pass else: raise InvalidRuleException("Unrecognised action") -- cgit 1.4.1 From ebc48306662cf8719a0ea64e2955bf8d5e037a8e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Mar 2015 09:53:00 +0000 Subject: PR tweaks: set earlier on and use 'as json' for compat --- synapse/storage/appservice.py | 18 +++++++----------- synapse/storage/registration.py | 2 +- synapse/storage/stream.py | 8 ++++---- 3 files changed, 12 insertions(+), 16 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index d0632d55d1..c6ca2ab04e 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -219,20 +219,17 @@ class ApplicationServiceStore(SQLBaseStore): # get all rooms matching the room ID regex. room_entries = yield self.get_all_rooms() # RoomEntry list - matching_room_id_list = [ + matching_room_list = set([ r.room_id for r in room_entries if service.is_interested_in_room(r.room_id) - ] + ]) # resolve room IDs for matching room alias regex. room_alias_mappings = yield self.get_all_associations() - matching_alias_list = [ + matching_room_list |= set([ r.room_id for r in room_alias_mappings if service.is_interested_in_alias(r.room_alias) - ] - room_ids_matching_alias_or_id = set( - matching_room_id_list + matching_alias_list - ) + ]) # get all rooms for every user for this AS. This is scoped to users on # this HS only. @@ -241,11 +238,10 @@ class ApplicationServiceStore(SQLBaseStore): u["name"] for u in user_list if service.is_interested_in_user(u["name"]) ] - rooms_for_user_matching_user_id = [] # RoomsForUser list + rooms_for_user_matching_user_id = set() # RoomsForUser list for user_id in user_list: rooms_for_user = yield self.get_rooms_for_user(user_id) - rooms_for_user_matching_user_id += rooms_for_user - rooms_for_user_matching_user_id = set(rooms_for_user_matching_user_id) + rooms_for_user_matching_user_id |= set(rooms_for_user) # make RoomsForUser tuples for room ids and aliases which are not in the # main rooms_for_user_list - e.g. they are rooms which do not have AS @@ -253,7 +249,7 @@ class ApplicationServiceStore(SQLBaseStore): known_room_ids = [r.room_id for r in rooms_for_user_matching_user_id] missing_rooms_for_user = [ RoomsForUser(r, service.sender, "join") for r in - room_ids_matching_alias_or_id if r not in known_room_ids + matching_room_list if r not in known_room_ids ] rooms_for_user_matching_user_id |= set(missing_rooms_for_user) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 7aff3dbd33..9c92575c7f 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -93,7 +93,7 @@ class RegistrationStore(SQLBaseStore): ) def get_all_users(self): - query = ("SELECT users.name FROM users") + query = "SELECT users.name FROM users" return self._execute( self.cursor_to_dict, query diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 09417bd147..bad427288d 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -43,7 +43,7 @@ from synapse.util.logutils import log_function from collections import namedtuple import logging -import simplejson +import simplejson as json logger = logging.getLogger(__name__) @@ -178,7 +178,7 @@ class StreamStore(SQLBaseStore): # interested in was invited to a room. We'll be passing this # through _get_events_txn later, so ignore the fact that this # may be a redacted event. - event_content = simplejson.loads(row["unrecognized_keys"]) + event_content = json.loads(row["unrecognized_keys"]) if (service.is_interested_in_user( event_content.get("state_key"))): return True @@ -202,7 +202,7 @@ class StreamStore(SQLBaseStore): self._set_before_and_after(ret, rows) if rows: - key = "s%d" % max([r["stream_ordering"] for r in rows]) + key = "s%d" % max(r["stream_ordering"] for r in rows) else: # Assume we didn't get anything because there was nothing to @@ -271,7 +271,7 @@ class StreamStore(SQLBaseStore): self._set_before_and_after(ret, rows) if rows: - key = "s%d" % max([r["stream_ordering"] for r in rows]) + key = "s%d" % max(r["stream_ordering"] for r in rows) else: # Assume we didn't get anything because there was nothing to -- cgit 1.4.1 From 3d73383d185b41b9986366da8123255e3a8ce1e0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Mar 2015 10:16:24 +0000 Subject: Modify _simple_select_list to allow an empty WHERE clause. Use it for get_all_rooms and get_all_users. --- synapse/storage/_base.py | 22 +++++++++++++++------- synapse/storage/appservice.py | 4 ++-- synapse/storage/registration.py | 7 ++----- synapse/storage/room.py | 5 ++--- 4 files changed, 21 insertions(+), 17 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c98dd36aed..3725c9795d 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -450,7 +450,8 @@ class SQLBaseStore(object): Args: table : string giving the table name - keyvalues : dict of column names and values to select the rows with + keyvalues : dict of column names and values to select the rows with, + or None to not apply a WHERE clause. retcols : list of strings giving the names of the columns to return """ return self.runInteraction( @@ -469,13 +470,20 @@ class SQLBaseStore(object): keyvalues : dict of column names and values to select the rows with retcols : list of strings giving the names of the columns to return """ - sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( - ", ".join(retcols), - table, - " AND ".join("%s = ?" % (k, ) for k in keyvalues) - ) + if keyvalues: + sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + ", ".join(retcols), + table, + " AND ".join("%s = ?" % (k, ) for k in keyvalues) + ) + txn.execute(sql, keyvalues.values()) + else: + sql = "SELECT %s FROM %s ORDER BY rowid asc" % ( + ", ".join(retcols), + table + ) + txn.execute(sql) - txn.execute(sql, keyvalues.values()) return self.cursor_to_dict(txn) def _simple_update_one(self, table, keyvalues, updatevalues, diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index c6ca2ab04e..0e3eab9422 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -220,8 +220,8 @@ class ApplicationServiceStore(SQLBaseStore): # get all rooms matching the room ID regex. room_entries = yield self.get_all_rooms() # RoomEntry list matching_room_list = set([ - r.room_id for r in room_entries if - service.is_interested_in_room(r.room_id) + r["room_id"] for r in room_entries if + service.is_interested_in_room(r["room_id"]) ]) # resolve room IDs for matching room alias regex. diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 9c92575c7f..54cd15bc0e 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -93,11 +93,8 @@ class RegistrationStore(SQLBaseStore): ) def get_all_users(self): - query = "SELECT users.name FROM users" - return self._execute( - self.cursor_to_dict, - query - ) + return self._simple_select_list( + table="users", keyvalues=None, retcols=["name"]) def get_user_by_token(self, token): """Get a user from the given access token. diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 3a64693404..6bd0b22ae5 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -77,9 +77,8 @@ class RoomStore(SQLBaseStore): Returns: A list of namedtuples containing the room information. """ - query = RoomsTable.select_statement() - return self._execute( - RoomsTable.decode_results, query, + return self._simple_select_list( + table="rooms", keyvalues=None, retcols=["room_id"] ) @defer.inlineCallbacks -- cgit 1.4.1 From b216b3689248094989168c340b60f500c93772a7 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Mar 2015 10:41:35 +0000 Subject: JOIN state_events rather than parsing unrecognized_keys to pull out member state_keys --- synapse/storage/appservice.py | 2 +- synapse/storage/stream.py | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 0e3eab9422..3a267d0442 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -218,7 +218,7 @@ class ApplicationServiceStore(SQLBaseStore): # less obvious. # get all rooms matching the room ID regex. - room_entries = yield self.get_all_rooms() # RoomEntry list + room_entries = yield self.get_all_rooms() matching_room_list = set([ r["room_id"] for r in room_entries if service.is_interested_in_room(r["room_id"]) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index bad427288d..865cb13e9e 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -43,7 +43,6 @@ from synapse.util.logutils import log_function from collections import namedtuple import logging -import simplejson as json logger = logging.getLogger(__name__) @@ -161,8 +160,9 @@ class StreamStore(SQLBaseStore): # select all the events between from/to with a sensible limit sql = ( - "SELECT e.event_id, e.room_id, e.type, e.unrecognized_keys, " - "e.stream_ordering FROM events AS e " + "SELECT e.event_id, e.room_id, e.type, s.state_key, " + "e.stream_ordering FROM events AS e LEFT JOIN state_events as s ON " + "e.event_id = s.event_id " "WHERE e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { @@ -174,13 +174,7 @@ class StreamStore(SQLBaseStore): return True if row["type"] == EventTypes.Member: - # load up the content to inspect if some user the AS is - # interested in was invited to a room. We'll be passing this - # through _get_events_txn later, so ignore the fact that this - # may be a redacted event. - event_content = json.loads(row["unrecognized_keys"]) - if (service.is_interested_in_user( - event_content.get("state_key"))): + if service.is_interested_in_user(row.get("state_key")): return True return False -- cgit 1.4.1 From 377ae369c1275fabdac46fa00c0b2ba238467435 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Mar 2015 11:20:51 +0000 Subject: Wrap all of get_app_service_rooms in a txn. --- synapse/storage/appservice.py | 38 +++++++++++++++++++++++----------- synapse/storage/directory.py | 21 ------------------- synapse/storage/registration.py | 4 ---- synapse/storage/room.py | 10 --------- synapse/storage/roommember.py | 36 +++++++++++++++++--------------- synapse/storage/stream.py | 46 ++++++++++++++++++++--------------------- 6 files changed, 67 insertions(+), 88 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 3a267d0442..97481d113b 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -15,6 +15,7 @@ import logging from twisted.internet import defer +from synapse.api.constants import Membership from synapse.api.errors import StoreError from synapse.appservice import ApplicationService from synapse.storage.roommember import RoomsForUser @@ -197,7 +198,6 @@ class ApplicationServiceStore(SQLBaseStore): # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. - @defer.inlineCallbacks def get_app_service_rooms(self, service): """Get a list of RoomsForUser for this application service. @@ -212,35 +212,49 @@ class ApplicationServiceStore(SQLBaseStore): Returns: A list of RoomsForUser. """ - # FIXME: This is assuming that this store has methods from - # RoomStore, DirectoryStore, RegistrationStore, RoomMemberStore which is - # a bad assumption to make as it makes testing trickier and coupling - # less obvious. + return self.runInteraction( + "get_app_service_rooms", + self._get_app_service_rooms_txn, + service, + ) + def _get_app_service_rooms_txn(self, txn, service): # get all rooms matching the room ID regex. - room_entries = yield self.get_all_rooms() + room_entries = self._simple_select_list_txn( + txn=txn, table="rooms", keyvalues=None, retcols=["room_id"] + ) matching_room_list = set([ r["room_id"] for r in room_entries if service.is_interested_in_room(r["room_id"]) ]) # resolve room IDs for matching room alias regex. - room_alias_mappings = yield self.get_all_associations() + room_alias_mappings = self._simple_select_list_txn( + txn=txn, table="room_aliases", keyvalues=None, + retcols=["room_id", "room_alias"] + ) matching_room_list |= set([ - r.room_id for r in room_alias_mappings if - service.is_interested_in_alias(r.room_alias) + r["room_id"] for r in room_alias_mappings if + service.is_interested_in_alias(r["room_alias"]) ]) # get all rooms for every user for this AS. This is scoped to users on # this HS only. - user_list = yield self.get_all_users() + user_list = self._simple_select_list_txn( + txn=txn, table="users", keyvalues=None, retcols=["name"] + ) user_list = [ u["name"] for u in user_list if service.is_interested_in_user(u["name"]) ] rooms_for_user_matching_user_id = set() # RoomsForUser list for user_id in user_list: - rooms_for_user = yield self.get_rooms_for_user(user_id) + # FIXME: This assumes this store is linked with RoomMemberStore :( + rooms_for_user = self._get_rooms_for_user_where_membership_is_txn( + txn=txn, + user_id=user_id, + membership_list=[Membership.JOIN] + ) rooms_for_user_matching_user_id |= set(rooms_for_user) # make RoomsForUser tuples for room ids and aliases which are not in the @@ -253,7 +267,7 @@ class ApplicationServiceStore(SQLBaseStore): ] rooms_for_user_matching_user_id |= set(missing_rooms_for_user) - defer.returnValue(rooms_for_user_matching_user_id) + return rooms_for_user_matching_user_id @defer.inlineCallbacks def _populate_cache(self): diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index e391239a3c..68b7d59693 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -134,27 +134,6 @@ class DirectoryStore(SQLBaseStore): return room_id - @defer.inlineCallbacks - def get_all_associations(self): - """Retrieve the entire list of room alias -> room ID pairings. - - Returns: - A list of RoomAliasMappings. - """ - results = yield self._execute_and_decode( - "SELECT room_id, room_alias FROM room_aliases" - ) - - # TODO(kegan): It feels wrong to be specifying no servers here, but - # equally this function isn't required to obtain all servers so - # retrieving them "just for the sake of it" also seems wrong, but we - # want to conform to passing Objects around and not dicts.. - defer.returnValue([ - RoomAliasMapping( - room_id=r["room_id"], room_alias=r["room_alias"], servers="" - ) for r in results - ]) - def get_aliases_for_room(self, room_id): return self._simple_select_onecol( "room_aliases", diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 54cd15bc0e..029b07cc66 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -92,10 +92,6 @@ class RegistrationStore(SQLBaseStore): query, user_id ) - def get_all_users(self): - return self._simple_select_list( - table="users", keyvalues=None, retcols=["name"]) - def get_user_by_token(self, token): """Get a user from the given access token. diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 6bd0b22ae5..750b17a45f 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -71,16 +71,6 @@ class RoomStore(SQLBaseStore): RoomsTable.decode_single_result, query, room_id, ) - def get_all_rooms(self): - """Retrieve all the rooms. - - Returns: - A list of namedtuples containing the room information. - """ - return self._simple_select_list( - table="rooms", keyvalues=None, retcols=["room_id"] - ) - @defer.inlineCallbacks def get_rooms(self, is_public): """Retrieve a list of all public rooms. diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 58aa376c20..3d0172d09b 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -180,6 +180,14 @@ class RoomMemberStore(SQLBaseStore): if not membership_list: return defer.succeed(None) + return self.runInteraction( + "get_rooms_for_user_where_membership_is", + self._get_rooms_for_user_where_membership_is_txn, + user_id, membership_list + ) + + def _get_rooms_for_user_where_membership_is_txn(self, txn, user_id, + membership_list): where_clause = "user_id = ? AND (%s)" % ( " OR ".join(["membership = ?" for _ in membership_list]), ) @@ -187,24 +195,18 @@ class RoomMemberStore(SQLBaseStore): args = [user_id] args.extend(membership_list) - def f(txn): - sql = ( - "SELECT m.room_id, m.sender, m.membership" - " FROM room_memberships as m" - " INNER JOIN current_state_events as c" - " ON m.event_id = c.event_id" - " WHERE %s" - ) % (where_clause,) - - txn.execute(sql, args) - return [ - RoomsForUser(**r) for r in self.cursor_to_dict(txn) - ] + sql = ( + "SELECT m.room_id, m.sender, m.membership" + " FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE %s" + ) % (where_clause,) - return self.runInteraction( - "get_rooms_for_user_where_membership_is", - f - ) + txn.execute(sql, args) + return [ + RoomsForUser(**r) for r in self.cursor_to_dict(txn) + ] def get_joined_hosts_for_room(self, room_id): return self._simple_select_onecol( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 865cb13e9e..09bc522210 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -146,18 +146,6 @@ class StreamStore(SQLBaseStore): defer.returnValue(([], to_key)) return - # Logic: - # - We want ALL events which match the AS room_id regex - # - We want ALL events which match the rooms represented by the AS - # room_alias regex - # - We want ALL events for rooms that AS users have joined. - # This is currently supported via get_app_service_rooms (which is used - # for the Notifier listener rooms). We can't reasonably make a SQL - # query for these room IDs, so we'll pull all the events between from/to - # and filter in python. - rooms_for_as = yield self.get_app_service_rooms(service) - room_ids_for_as = [r.room_id for r in rooms_for_as] - # select all the events between from/to with a sensible limit sql = ( "SELECT e.event_id, e.room_id, e.type, s.state_key, " @@ -169,20 +157,32 @@ class StreamStore(SQLBaseStore): "limit": limit } - def app_service_interested(row): - if row["room_id"] in room_ids_for_as: - return True - - if row["type"] == EventTypes.Member: - if service.is_interested_in_user(row.get("state_key")): - return True - return False - def f(txn): + # pull out all the events between the tokens txn.execute(sql, (from_id.stream, to_id.stream,)) - rows = self.cursor_to_dict(txn) + # Logic: + # - We want ALL events which match the AS room_id regex + # - We want ALL events which match the rooms represented by the AS + # room_alias regex + # - We want ALL events for rooms that AS users have joined. + # This is currently supported via get_app_service_rooms (which is + # used for the Notifier listener rooms). We can't reasonably make a + # SQL query for these room IDs, so we'll pull all the events between + # from/to and filter in python. + rooms_for_as = self._get_app_service_rooms_txn(txn, service) + room_ids_for_as = [r.room_id for r in rooms_for_as] + + def app_service_interested(row): + if row["room_id"] in room_ids_for_as: + return True + + if row["type"] == EventTypes.Member: + if service.is_interested_in_user(row.get("state_key")): + return True + return False + ret = self._get_events_txn( txn, # apply the filter on the room id list @@ -197,7 +197,6 @@ class StreamStore(SQLBaseStore): if rows: key = "s%d" % max(r["stream_ordering"] for r in rows) - else: # Assume we didn't get anything because there was nothing to # get. @@ -266,7 +265,6 @@ class StreamStore(SQLBaseStore): if rows: key = "s%d" % max(r["stream_ordering"] for r in rows) - else: # Assume we didn't get anything because there was nothing to # get. -- cgit 1.4.1 From cb97ea3ec236c23c745e59c3a857503dd8dc3410 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Mar 2015 11:23:46 +0000 Subject: PEP8 --- synapse/storage/roommember.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 3d0172d09b..65ffb4627f 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -187,7 +187,7 @@ class RoomMemberStore(SQLBaseStore): ) def _get_rooms_for_user_where_membership_is_txn(self, txn, user_id, - membership_list): + membership_list): where_clause = "user_id = ? AND (%s)" % ( " OR ".join(["membership = ?" for _ in membership_list]), ) -- cgit 1.4.1 From 9d9b230501915c326136567349b0995623c48a21 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Mar 2015 11:33:45 +0000 Subject: Make the federation server ratelimiting configurable. --- synapse/config/ratelimiting.py | 36 ++++++++++++++++++++++++++++++++ synapse/federation/transport/__init__.py | 10 ++++----- 2 files changed, 41 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 17c7e64ce7..862c07ef8c 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -22,6 +22,12 @@ class RatelimitConfig(Config): self.rc_messages_per_second = args.rc_messages_per_second self.rc_message_burst_count = args.rc_message_burst_count + self.federation_rc_window_size = args.federation_rc_window_size + self.federation_rc_sleep_limit = args.federation_rc_sleep_limit + self.federation_rc_sleep_delay = args.federation_rc_sleep_delay + self.federation_rc_reject_limit = args.federation_rc_reject_limit + self.federation_rc_concurrent = args.federation_rc_concurrent + @classmethod def add_arguments(cls, parser): super(RatelimitConfig, cls).add_arguments(parser) @@ -34,3 +40,33 @@ class RatelimitConfig(Config): "--rc-message-burst-count", type=float, default=10, help="number of message a client can send before being throttled" ) + + rc_group.add_argument( + "--federation-rc-window-size", type=int, default=10000, + help="The federation window size in milliseconds", + ) + + rc_group.add_argument( + "--federation-rc-sleep-limit", type=int, default=10, + help="The number of federation requests from a single server" + " in a window before the server will delay processing the" + " request.", + ) + + rc_group.add_argument( + "--federation-rc-sleep-delay", type=int, default=500, + help="The duration in milliseconds to delay processing events from" + " remote servers by if they go over the sleep limit.", + ) + + rc_group.add_argument( + "--federation-rc-reject-limit", type=int, default=50, + help="The maximum number of concurrent federation requests allowed" + " from a single server", + ) + + rc_group.add_argument( + "--federation-rc-concurrent", type=int, default=3, + help="The number of federation requests to concurrently process" + " from a single server", + ) diff --git a/synapse/federation/transport/__init__.py b/synapse/federation/transport/__init__.py index f0283b5105..2a671b9aec 100644 --- a/synapse/federation/transport/__init__.py +++ b/synapse/federation/transport/__init__.py @@ -66,9 +66,9 @@ class TransportLayer(TransportLayerServer, TransportLayerClient): self.ratelimiter = FederationRateLimiter( self.clock, - window_size=10000, - sleep_limit=10, - sleep_msec=500, - reject_limit=50, - concurrent_requests=3, + window_size=homeserver.config.federation_rc_window_size, + sleep_limit=homeserver.config.federation_rc_sleep_limit, + sleep_msec=homeserver.config.federation_rc_sleep_delay, + reject_limit=homeserver.config.federation_rc_reject_limit, + concurrent_requests=homeserver.config.federation_rc_concurrent, ) -- cgit 1.4.1 From 23d9bd1d745a037202bb9a134cdb848eb65a01e9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Mar 2015 11:39:40 +0000 Subject: Process transactions serially. Since the events received in a transaction are ordered, later events might depend on earlier events and so we shouldn't blindly process them in parellel. --- synapse/federation/federation_server.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 22b9663831..7ee37fb34d 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -112,17 +112,23 @@ class FederationServer(FederationBase): logger.debug("[%s] Transaction is new", transaction.transaction_id) with PreserveLoggingContext(): - dl = [] + results = [] + for pdu in pdu_list: d = self._handle_new_pdu(transaction.origin, pdu) def handle_failure(failure): failure.trap(FederationError) self.send_failure(failure.value, transaction.origin) + return failure d.addErrback(handle_failure) - dl.append(d) + try: + yield d + results.append({}) + except Exception as e: + results.append({"error": str(e)}) if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: @@ -135,21 +141,11 @@ class FederationServer(FederationBase): for failure in getattr(transaction, "pdu_failures", []): logger.info("Got failure %r", failure) - results = yield defer.DeferredList(dl, consumeErrors=True) - - ret = [] - for r in results: - if r[0]: - ret.append({}) - else: - logger.exception(r[1]) - ret.append({"error": str(r[1].value)}) - - logger.debug("Returning: %s", str(ret)) + logger.debug("Returning: %s", str(results)) response = { "pdus": dict(zip( - (p.event_id for p in pdu_list), ret + (p.event_id for p in pdu_list), results )), } -- cgit 1.4.1 From 29481690c5b296a1c8aee3068d32ef083ef227f3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Mar 2015 11:50:43 +0000 Subject: If we're yielding don't add errback --- synapse/federation/federation_server.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 7ee37fb34d..bc9bac809a 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -117,16 +117,12 @@ class FederationServer(FederationBase): for pdu in pdu_list: d = self._handle_new_pdu(transaction.origin, pdu) - def handle_failure(failure): - failure.trap(FederationError) - self.send_failure(failure.value, transaction.origin) - return failure - - d.addErrback(handle_failure) - try: yield d results.append({}) + except FederationError as e: + self.send_failure(e, transaction.origin) + results.append({"error": str(e)}) except Exception as e: results.append({"error": str(e)}) -- cgit 1.4.1 From 769f8b58e8023c5e0a5b6970c27d2c0ed223a8af Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 2 Mar 2015 13:28:24 +0000 Subject: Rename the room-with-two-people rule to be more compatible if we have actual one to one rooms. --- synapse/push/baserules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 09878b63c1..c8c19f2a18 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -84,7 +84,7 @@ def make_base_override_rules(): ] }, { - 'rule_id': 'global/override/.m.rule.room_two_members', + 'rule_id': 'global/override/.m.rule.room_one_to_one', 'conditions': [ { 'kind': 'room_member_count', -- cgit 1.4.1 From 3077cb291590a7ba3b24a3d1a9985d65980924fb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Mar 2015 13:32:44 +0000 Subject: Use contextlib.contextmanager instead of a custom class --- synapse/util/ratelimitutils.py | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) (limited to 'synapse') diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 259d5f6f88..d4457af950 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -20,6 +20,7 @@ from synapse.api.errors import LimitExceededError from synapse.util.async import sleep import collections +import contextlib import logging @@ -112,16 +113,19 @@ class _PerHostRatelimiter(object): or self.request_times ) + @contextlib.contextmanager def ratelimit(self): - request_id = object() - - def on_enter(): - return self._on_enter(request_id) - - def on_exit(exc_type, exc_val, exc_tb): - return self._on_exit(request_id) + # `contextlib.contextmanager` takes a generator and turns it into a + # context manager. The generator should only yield once with a value + # to be returned by manager. + # Exceptions will be reraised at the yield. - return ContextManagerFunction(on_enter, on_exit) + request_id = object() + ret = self._on_enter(request_id) + try: + yield ret + finally: + self._on_exit(request_id) def _on_enter(self, request_id): time_now = self.clock.time_msec() @@ -210,17 +214,3 @@ class _PerHostRatelimiter(object): deferred.callback(None) except KeyError: pass - - -class ContextManagerFunction(object): - def __init__(self, on_enter, on_exit): - self.on_enter = on_enter - self.on_exit = on_exit - - def __enter__(self): - if self.on_enter: - return self.on_enter() - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.on_exit: - return self.on_exit(exc_type, exc_val, exc_tb) -- cgit 1.4.1 From c3c01641d2d49988c59826912e4e48740ab4f32a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Mar 2015 13:38:57 +0000 Subject: Run deltas and bump user_version in upgrade script --- UPGRADE.rst | 5 +++++ scripts/upgrade_appservice_db.py | 28 +++++++++++++++++++++++----- synapse/storage/__init__.py | 2 +- 3 files changed, 29 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/UPGRADE.rst b/UPGRADE.rst index 4045baf4e0..8cda8d02a0 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -12,6 +12,11 @@ Servers which use captchas will need to add their public key to:: This is required in order to support registration fallback (typically used on mobile devices). +Servers which have registered application services need to upgrade their +database as the format of stored application services has changed in Synapse. +Run ``python upgrade_appservice_db.py `` to convert to the +new format. + Upgrading to v0.7.0 =================== diff --git a/scripts/upgrade_appservice_db.py b/scripts/upgrade_appservice_db.py index acdee56d9f..ae1b91c64f 100644 --- a/scripts/upgrade_appservice_db.py +++ b/scripts/upgrade_appservice_db.py @@ -1,17 +1,28 @@ +from synapse.storage import read_schema import argparse import json import sqlite3 -def main(dbname): - con = sqlite3.connect(dbname) - cur = con.cursor() +def do_other_deltas(cursor): + cursor.execute("PRAGMA user_version") + row = cursor.fetchone() + + if row and row[0]: + user_version = row[0] + # Run every version since after the current version. + for v in range(user_version + 1, 10): + print "Running delta: %d" % (v,) + sql_script = read_schema("delta/v%d" % (v,)) + cursor.executescript(sql_script) + + +def update_app_service_table(cur): cur.execute("SELECT id, regex FROM application_services_regex") for row in cur.fetchall(): try: print "checking %s..." % row[0] json.loads(row[1]) - print "Already in new format" except ValueError: # row isn't in json, make it so. string_regex = row[1] @@ -23,13 +34,20 @@ def main(dbname): "UPDATE application_services_regex SET regex=? WHERE id=?", (new_regex, row[0]) ) + + +def main(dbname): + con = sqlite3.connect(dbname) + cur = con.cursor() + do_other_deltas(cur) + update_app_service_table(cur) + cur.execute("PRAGMA user_version = 14") cur.close() con.commit() if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("database") args = parser.parse_args() diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d16e7b8fac..3753cd28d0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -74,7 +74,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 13 +SCHEMA_VERSION = 14 dir_path = os.path.abspath(os.path.dirname(__file__)) -- cgit 1.4.1 From 8486910b64ba55292bfb7c619984eb73fd0a7e14 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Mar 2015 14:57:04 +0000 Subject: Bump webclient version --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index c71df75404..5fe8a825e3 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil>=0.0.3": ["syutil"], - "matrix_angular_sdk>=0.6.3": ["syweb>=0.6.3"], + "matrix_angular_sdk>=0.6.4": ["syweb>=0.6.4"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -36,8 +36,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/matrix-angular-sdk", - version="v0.6.3", - egg="matrix_angular_sdk-0.6.3", + version="v0.6.4", + egg="matrix_angular_sdk-0.6.4", ), ] -- cgit 1.4.1 From 2de5b14fe00d2337a3179aa19185e66350ba1a3e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Mar 2015 15:36:37 +0000 Subject: Fix bug which prevented the HS pushing events to the AS due to FrozenEvents --- synapse/http/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/http/client.py b/synapse/http/client.py index 22b7145ac1..b53a07aa2d 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -143,7 +143,7 @@ class SimpleHttpClient(object): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) - json_str = json.dumps(json_body) + json_str = encode_canonical_json(json_body) response = yield self.agent.request( "PUT", -- cgit 1.4.1 From 20436cdf7522fdb76e8d883cf251d9332c0ea6d3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 2 Mar 2015 15:58:12 +0000 Subject: Blank lines --- synapse/rest/client/v1/push_rule.py | 1 + synapse/storage/push_rule.py | 1 + 2 files changed, 2 insertions(+) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 5bfdb29907..3db38a949a 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -218,6 +218,7 @@ class PushRuleRestServlet(ClientV1RestServlet): else: raise UnrecognizedRequestError() + def _rule_spec_from_path(path): if len(path) < 2: raise UnrecognizedRequestError() diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 6c38565773..c648c9960d 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -229,6 +229,7 @@ class PushRuleStore(SQLBaseStore): {'enabled': False} ) + class RuleNotFoundException(Exception): pass -- cgit 1.4.1 From b41dc687739e92d5debcae5bb9feb21b97cbc178 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Mar 2015 16:36:19 +0000 Subject: We purposefully don't have a version 14 delta script. --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 3753cd28d0..f0b7b8fef3 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -633,7 +633,7 @@ def prepare_database(db_conn): # Run every version since after the current version. for v in range(user_version + 1, SCHEMA_VERSION + 1): - if v == 10: + if v in (10, 14,): raise UpgradeDatabaseException( "No delta for version 10" ) -- cgit 1.4.1 From 9f03553f48d774ba88ca91a14050c94c4a1b77b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Mar 2015 16:38:40 +0000 Subject: Add missing comma --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f0b7b8fef3..d6ec446bd2 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -637,7 +637,7 @@ def prepare_database(db_conn): raise UpgradeDatabaseException( "No delta for version 10" ) - sql_script = read_schema("delta/v%d" % (v)) + sql_script = read_schema("delta/v%d" % (v,)) c.executescript(sql_script) db_conn.commit() -- cgit 1.4.1 From 09f9e8493c574b1c0c120e153cda4beebafce7dd Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 2 Mar 2015 17:37:22 +0000 Subject: Oops, missed a replacement. --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 3db38a949a..822c978e85 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -387,7 +387,7 @@ def _priority_class_from_spec(spec): def _priority_class_to_template_name(pc): if pc > PRIORITY_CLASS_MAP['override']: # per-device - prio_class_index = pc - len(PushRuleRestServlet.PRIORITY_CLASS_MAP) + prio_class_index = pc - len(PRIORITY_CLASS_MAP) return PRIORITY_CLASS_INVERSE_MAP[prio_class_index] else: return PRIORITY_CLASS_INVERSE_MAP[pc] -- cgit 1.4.1 From 6fab7bd2c1102c3f3254074cc996d950805531b4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 2 Mar 2015 18:17:19 +0000 Subject: s/user_name/user/ as per mjark's comment --- synapse/push/__init__.py | 4 ++-- synapse/rest/client/v1/push_rule.py | 6 +++--- synapse/storage/push_rule.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index d4da05f093..ba1aac30fb 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -76,13 +76,13 @@ class Pusher(object): if ev['state_key'] != self.user_name: defer.returnValue(['dont_notify']) - rawrules = yield self.store.get_push_rules_for_user_name(self.user_name) + rawrules = yield self.store.get_push_rules_for_user(self.user_name) for r in rawrules: r['conditions'] = json.loads(r['conditions']) r['actions'] = json.loads(r['actions']) - enabled_map = yield self.store.get_push_rules_enabled_for_user_name(self.user_name) + enabled_map = yield self.store.get_push_rules_enabled_for_user(self.user_name) user = UserID.from_string(self.user_name) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 822c978e85..fef0eb6572 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -114,7 +114,7 @@ class PushRuleRestServlet(ClientV1RestServlet): # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference - rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name( + rawrules = yield self.hs.get_datastore().get_push_rules_for_user( user.to_string() ) @@ -129,7 +129,7 @@ class PushRuleRestServlet(ClientV1RestServlet): rules['global'] = _add_empty_priority_class_arrays(rules['global']) enabled_map = yield self.hs.get_datastore().\ - get_push_rules_enabled_for_user_name(user.to_string()) + get_push_rules_enabled_for_user(user.to_string()) for r in ruleslist: rulearray = None @@ -212,7 +212,7 @@ class PushRuleRestServlet(ClientV1RestServlet): def get_rule_attr(self, user_name, namespaced_rule_id, attr): if attr == 'enabled': - return self.hs.get_datastore().get_push_rule_enabled_by_user_name_rule_id( + return self.hs.get_datastore().get_push_rule_enabled_by_user_rule_id( user_name, namespaced_rule_id ) else: diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index c648c9960d..ea865b6abf 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) class PushRuleStore(SQLBaseStore): @defer.inlineCallbacks - def get_push_rules_for_user_name(self, user_name): + def get_push_rules_for_user(self, user_name): sql = ( "SELECT "+",".join(PushRuleTable.fields)+" " "FROM "+PushRuleTable.table_name+" " @@ -46,7 +46,7 @@ class PushRuleStore(SQLBaseStore): defer.returnValue(dicts) @defer.inlineCallbacks - def get_push_rules_enabled_for_user_name(self, user_name): + def get_push_rules_enabled_for_user(self, user_name): results = yield self._simple_select_list( PushRuleEnableTable.table_name, {'user_name': user_name}, @@ -57,7 +57,7 @@ class PushRuleStore(SQLBaseStore): ) @defer.inlineCallbacks - def get_push_rule_enabled_by_user_name_rule_id(self, user_name, rule_id): + def get_push_rule_enabled_by_user_rule_id(self, user_name, rule_id): results = yield self._simple_select_list( PushRuleEnableTable.table_name, {'user_name': user_name, 'rule_id': rule_id}, -- cgit 1.4.1 From 8d33adfbbbd01a9a85e862ad2bd21ae45230e710 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Mar 2015 18:23:55 +0000 Subject: SYN-67: Begin changing the way we handle schema versioning --- synapse/storage/__init__.py | 144 ++++++++++++++++++++++++++++++++------------ 1 file changed, 106 insertions(+), 38 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d6ec446bd2..a08c74fac1 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -45,8 +45,10 @@ from syutil.jsonutil import encode_canonical_json from synapse.crypto.event_signing import compute_event_reference_hash +import imp import logging import os +import sqlite3 logger = logging.getLogger(__name__) @@ -610,49 +612,115 @@ class UpgradeDatabaseException(PrepareDatabaseException): def prepare_database(db_conn): - """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we - don't have to worry about overwriting existing content. + """Prepares a database for usage. Will either create all necessary tables + or upgrade from an older schema version. """ - c = db_conn.cursor() - c.execute("PRAGMA user_version") - row = c.fetchone() + cur = db_conn.cursor() + version_info = get_schema_state(cur) - if row and row[0]: - user_version = row[0] + if version_info: + user_version, delta_files = version_info + _upgrade_existing_database(cur, user_version, delta_files) + else: + _setup_new_database(cur) + + cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) + db_conn.commit() + + cur.close() + + +def _setup_new_database(cur): + sql_script = "BEGIN TRANSACTION;\n" + for sql_loc in SCHEMAS: + logger.debug("Applying schema %r", sql_loc) + sql_script += read_schema(sql_loc) + sql_script += "\n" + sql_script += "COMMIT TRANSACTION;" + cur.executescript(sql_script) + + +def _upgrade_existing_database(cur, user_version, delta_files): + """Upgrades an existing database. + + Delta files can either be SQL stored in *.sql files, or python modules + in *.py. + + There can be multiple delta files per version. Synapse will keep track of + which delta files have been applied, and will apply any that haven't been + even if there has been no version bump. This is useful for development + where orthogonal schema changes may happen on separate branches. + """ + + if user_version > SCHEMA_VERSION: + raise ValueError( + "Cannot use this database as it is too " + + "new for the server to understand" + ) - if user_version > SCHEMA_VERSION: - raise ValueError( - "Cannot use this database as it is too " + - "new for the server to understand" + for v in range(user_version, SCHEMA_VERSION + 1): + delta_dir = os.path.join(dir_path, "schema", "delta", v) + directory_entries = os.listdir(delta_dir) + + for file_name in directory_entries: + relative_path = os.path.join(v, file_name) + if relative_path in delta_files: + continue + + absolute_path = os.path.join( + dir_path, "schema", "delta", relative_path, ) - elif user_version < SCHEMA_VERSION: - logger.info( - "Upgrading database from version %d", - user_version + root_name, ext = os.path.splitext(file_name) + if ext == ".py": + module_name = "synapse.storage.schema.v%d_%s" % ( + v, root_name + ) + with open(absolute_path) as schema_file: + module = imp.load_source( + module_name, absolute_path, schema_file + ) + module.run_upgrade(cur) + elif ext == ".sql": + with open(absolute_path) as schema_file: + delta_schema = schema_file.read() + cur.executescript(delta_schema) + else: + # Not a valid delta file. + logger.warn( + "Found directory entry that did not end in .py or" + " .sql: %s", + relative_path, + ) + continue + + # Mark as done. + cur.execute( + "INSERT INTO schema_version (version, file)" + " VALUES (?,?)", + (v, relative_path) ) - # Run every version since after the current version. - for v in range(user_version + 1, SCHEMA_VERSION + 1): - if v in (10, 14,): - raise UpgradeDatabaseException( - "No delta for version 10" - ) - sql_script = read_schema("delta/v%d" % (v,)) - c.executescript(sql_script) - db_conn.commit() - else: - logger.info("Database is at version %r", user_version) +def get_schema_state(txn): + sql = ( + "SELECT MAX(version), file FROM schema_version" + " WHERE version = (SELECT MAX(version) FROM schema_version)" + ) - else: - sql_script = "BEGIN TRANSACTION;\n" - for sql_loc in SCHEMAS: - logger.debug("Applying schema %r", sql_loc) - sql_script += read_schema(sql_loc) - sql_script += "\n" - sql_script += "COMMIT TRANSACTION;" - c.executescript(sql_script) - db_conn.commit() - c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION) - - c.close() + try: + txn.execute(sql) + res = txn.fetchall() + + if res: + current_verison = max(r[0] for r in res) + applied_delta = [r[1] for r in res] + + return current_verison, applied_delta + except sqlite3.OperationalError: + txn.execute("PRAGMA user_version") + row = txn.fetchone() + if row and row[0]: + # FIXME: We need to create schema_version table! + return row[0], [] + + return None -- cgit 1.4.1 From da877aad1560e859f5724a89a59a21aaf061e3de Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Mar 2015 13:31:50 +0000 Subject: Bump syweb dependency --- synapse/__init__.py | 2 +- synapse/python_dependencies.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 7d548b0a8a..f78a67454d 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.1-r2" +__version__ = "0.7.1-r3" diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index c71df75404..1195f2cd06 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil>=0.0.3": ["syutil"], - "matrix_angular_sdk>=0.6.3": ["syweb>=0.6.3"], + "matrix_angular_sdk>=0.6.4": ["syweb>=0.6.4"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], -- cgit 1.4.1 From ede89ae3b420fcef0bfc129ec104d3ccd99f8b94 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Mar 2015 14:49:19 +0000 Subject: Also bump version of downloaded syweb --- synapse/python_dependencies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 1195f2cd06..5fe8a825e3 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -36,8 +36,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/matrix-angular-sdk", - version="v0.6.3", - egg="matrix_angular_sdk-0.6.3", + version="v0.6.4", + egg="matrix_angular_sdk-0.6.4", ), ] -- cgit 1.4.1 From 527e0c43a53febfe5bc1b5480927d3304a16cb85 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Mar 2015 14:49:34 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index f78a67454d..74c5584a1a 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.1-r3" +__version__ = "0.7.1-r4" -- cgit 1.4.1 From 82b34e813de4dadb8ec5bce068f7113e32e60ead Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 12:04:19 +0000 Subject: SYN-67: Finish up implementing new database schema management --- scripts/upgrade_appservice_db.py | 54 ------ synapse/app/homeserver.py | 5 +- synapse/storage/__init__.py | 197 ++++++++++++++------- synapse/storage/schema/application_services.sql | 34 ---- synapse/storage/schema/current/11/event_edges.sql | 89 ++++++++++ .../storage/schema/current/11/event_signatures.sql | 65 +++++++ synapse/storage/schema/current/11/im.sql | 125 +++++++++++++ synapse/storage/schema/current/11/keys.sql | 31 ++++ .../storage/schema/current/11/media_repository.sql | 68 +++++++ synapse/storage/schema/current/11/presence.sql | 38 ++++ synapse/storage/schema/current/11/profiles.sql | 20 +++ synapse/storage/schema/current/11/redactions.sql | 22 +++ synapse/storage/schema/current/11/room_aliases.sql | 27 +++ synapse/storage/schema/current/11/state.sql | 47 +++++ synapse/storage/schema/current/11/transactions.sql | 68 +++++++ synapse/storage/schema/current/11/users.sql | 45 +++++ synapse/storage/schema/delta/11/v11.sql | 16 ++ synapse/storage/schema/delta/12/v12.sql | 67 +++++++ synapse/storage/schema/delta/13/v13.sql | 34 ++++ .../schema/delta/14/upgrade_appservice_db.py | 20 +++ synapse/storage/schema/delta/v11.sql | 16 -- synapse/storage/schema/delta/v12.sql | 67 ------- synapse/storage/schema/delta/v13.sql | 34 ---- synapse/storage/schema/delta/v2.sql | 168 ------------------ synapse/storage/schema/delta/v3.sql | 27 --- synapse/storage/schema/delta/v4.sql | 26 --- synapse/storage/schema/delta/v5.sql | 30 ---- synapse/storage/schema/delta/v6.sql | 31 ---- synapse/storage/schema/delta/v8.sql | 34 ---- synapse/storage/schema/delta/v9.sql | 79 --------- synapse/storage/schema/event_edges.sql | 89 ---------- synapse/storage/schema/event_signatures.sql | 65 ------- synapse/storage/schema/filtering.sql | 24 --- synapse/storage/schema/im.sql | 125 ------------- synapse/storage/schema/keys.sql | 31 ---- synapse/storage/schema/media_repository.sql | 68 ------- synapse/storage/schema/presence.sql | 38 ---- synapse/storage/schema/profiles.sql | 20 --- synapse/storage/schema/pusher.sql | 46 ----- synapse/storage/schema/redactions.sql | 22 --- synapse/storage/schema/rejections.sql | 21 --- synapse/storage/schema/room_aliases.sql | 27 --- synapse/storage/schema/schema_version.sql | 29 +++ synapse/storage/schema/state.sql | 47 ----- synapse/storage/schema/transactions.sql | 68 ------- synapse/storage/schema/users.sql | 45 ----- 46 files changed, 946 insertions(+), 1403 deletions(-) delete mode 100644 scripts/upgrade_appservice_db.py delete mode 100644 synapse/storage/schema/application_services.sql create mode 100644 synapse/storage/schema/current/11/event_edges.sql create mode 100644 synapse/storage/schema/current/11/event_signatures.sql create mode 100644 synapse/storage/schema/current/11/im.sql create mode 100644 synapse/storage/schema/current/11/keys.sql create mode 100644 synapse/storage/schema/current/11/media_repository.sql create mode 100644 synapse/storage/schema/current/11/presence.sql create mode 100644 synapse/storage/schema/current/11/profiles.sql create mode 100644 synapse/storage/schema/current/11/redactions.sql create mode 100644 synapse/storage/schema/current/11/room_aliases.sql create mode 100644 synapse/storage/schema/current/11/state.sql create mode 100644 synapse/storage/schema/current/11/transactions.sql create mode 100644 synapse/storage/schema/current/11/users.sql create mode 100644 synapse/storage/schema/delta/11/v11.sql create mode 100644 synapse/storage/schema/delta/12/v12.sql create mode 100644 synapse/storage/schema/delta/13/v13.sql create mode 100644 synapse/storage/schema/delta/14/upgrade_appservice_db.py delete mode 100644 synapse/storage/schema/delta/v11.sql delete mode 100644 synapse/storage/schema/delta/v12.sql delete mode 100644 synapse/storage/schema/delta/v13.sql delete mode 100644 synapse/storage/schema/delta/v2.sql delete mode 100644 synapse/storage/schema/delta/v3.sql delete mode 100644 synapse/storage/schema/delta/v4.sql delete mode 100644 synapse/storage/schema/delta/v5.sql delete mode 100644 synapse/storage/schema/delta/v6.sql delete mode 100644 synapse/storage/schema/delta/v8.sql delete mode 100644 synapse/storage/schema/delta/v9.sql delete mode 100644 synapse/storage/schema/event_edges.sql delete mode 100644 synapse/storage/schema/event_signatures.sql delete mode 100644 synapse/storage/schema/filtering.sql delete mode 100644 synapse/storage/schema/im.sql delete mode 100644 synapse/storage/schema/keys.sql delete mode 100644 synapse/storage/schema/media_repository.sql delete mode 100644 synapse/storage/schema/presence.sql delete mode 100644 synapse/storage/schema/profiles.sql delete mode 100644 synapse/storage/schema/pusher.sql delete mode 100644 synapse/storage/schema/redactions.sql delete mode 100644 synapse/storage/schema/rejections.sql delete mode 100644 synapse/storage/schema/room_aliases.sql create mode 100644 synapse/storage/schema/schema_version.sql delete mode 100644 synapse/storage/schema/state.sql delete mode 100644 synapse/storage/schema/transactions.sql delete mode 100644 synapse/storage/schema/users.sql (limited to 'synapse') diff --git a/scripts/upgrade_appservice_db.py b/scripts/upgrade_appservice_db.py deleted file mode 100644 index ae1b91c64f..0000000000 --- a/scripts/upgrade_appservice_db.py +++ /dev/null @@ -1,54 +0,0 @@ -from synapse.storage import read_schema -import argparse -import json -import sqlite3 - - -def do_other_deltas(cursor): - cursor.execute("PRAGMA user_version") - row = cursor.fetchone() - - if row and row[0]: - user_version = row[0] - # Run every version since after the current version. - for v in range(user_version + 1, 10): - print "Running delta: %d" % (v,) - sql_script = read_schema("delta/v%d" % (v,)) - cursor.executescript(sql_script) - - -def update_app_service_table(cur): - cur.execute("SELECT id, regex FROM application_services_regex") - for row in cur.fetchall(): - try: - print "checking %s..." % row[0] - json.loads(row[1]) - except ValueError: - # row isn't in json, make it so. - string_regex = row[1] - new_regex = json.dumps({ - "regex": string_regex, - "exclusive": True - }) - cur.execute( - "UPDATE application_services_regex SET regex=? WHERE id=?", - (new_regex, row[0]) - ) - - -def main(dbname): - con = sqlite3.connect(dbname) - cur = con.cursor() - do_other_deltas(cur) - update_app_service_table(cur) - cur.execute("PRAGMA user_version = 14") - cur.close() - con.commit() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("database") - args = parser.parse_args() - - main(args.database) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5695d5aff8..b3ba7dfddc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,9 @@ import sys sys.dont_write_bytecode = True -from synapse.storage import prepare_database, UpgradeDatabaseException +from synapse.storage import ( + prepare_database, prepare_sqlite3_database, UpgradeDatabaseException, +) from synapse.server import HomeServer @@ -335,6 +337,7 @@ def setup(): try: with sqlite3.connect(db_name) as db_conn: + prepare_sqlite3_database(db_conn) prepare_database(db_conn) except UpgradeDatabaseException: sys.stderr.write( diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a08c74fac1..07ccc4e2ee 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -45,35 +45,16 @@ from syutil.jsonutil import encode_canonical_json from synapse.crypto.event_signing import compute_event_reference_hash +import fnmatch import imp import logging import os -import sqlite3 +import re logger = logging.getLogger(__name__) -SCHEMAS = [ - "transactions", - "users", - "profiles", - "presence", - "im", - "room_aliases", - "keys", - "redactions", - "state", - "event_edges", - "event_signatures", - "pusher", - "media_repository", - "application_services", - "filtering", - "rejections", -] - - # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. SCHEMA_VERSION = 14 @@ -578,28 +559,15 @@ class DataStore(RoomMemberStore, RoomStore, ) -def schema_path(schema): - """ Get a filesystem path for the named database schema - - Args: - schema: Name of the database schema. - Returns: - A filesystem path pointing at a ".sql" file. - - """ - schemaPath = os.path.join(dir_path, "schema", schema + ".sql") - return schemaPath - - -def read_schema(schema): +def read_schema(path): """ Read the named database schema. Args: - schema: Name of the datbase schema. + path: Path of the database schema. Returns: A string containing the database schema. """ - with open(schema_path(schema)) as schema_file: + with open(path) as schema_file: return schema_file.read() @@ -616,11 +584,11 @@ def prepare_database(db_conn): or upgrade from an older schema version. """ cur = db_conn.cursor() - version_info = get_schema_state(cur) + version_info = get_or_create_schema_state(cur) if version_info: - user_version, delta_files = version_info - _upgrade_existing_database(cur, user_version, delta_files) + user_version, delta_files, upgraded = version_info + _upgrade_existing_database(cur, user_version, delta_files, upgraded) else: _setup_new_database(cur) @@ -631,16 +599,52 @@ def prepare_database(db_conn): def _setup_new_database(cur): + current_dir = os.path.join(dir_path, "schema", "current") + directory_entries = os.listdir(current_dir) + + valid_dirs = [] + pattern = re.compile(r"^\d+(\.sql)?$") + for filename in directory_entries: + match = pattern.match(filename) + abs_path = os.path.join(current_dir, filename) + if match and os.path.isdir(abs_path): + ver = int(match.group(0)) + if ver < SCHEMA_VERSION: + valid_dirs.append((ver, abs_path)) + + if not valid_dirs: + raise RuntimeError("Could not find a suitable current.sql") + + max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0]) + + logger.debug("Initialising schema v%d", max_current_ver) + + directory_entries = os.listdir(sql_dir) + sql_script = "BEGIN TRANSACTION;\n" - for sql_loc in SCHEMAS: + for filename in fnmatch.filter(directory_entries, "*.sql"): + sql_loc = os.path.join(sql_dir, filename) logger.debug("Applying schema %r", sql_loc) sql_script += read_schema(sql_loc) sql_script += "\n" sql_script += "COMMIT TRANSACTION;" cur.executescript(sql_script) + cur.execute( + "INSERT INTO schema_version (version, upgraded)" + " VALUES (?,?)", + (max_current_ver, False) + ) + + _upgrade_existing_database( + cur, + current_version=max_current_ver, + delta_files=[], + upgraded=False + ) + -def _upgrade_existing_database(cur, user_version, delta_files): +def _upgrade_existing_database(cur, current_version, delta_files, upgraded): """Upgrades an existing database. Delta files can either be SQL stored in *.sql files, or python modules @@ -650,20 +654,41 @@ def _upgrade_existing_database(cur, user_version, delta_files): which delta files have been applied, and will apply any that haven't been even if there has been no version bump. This is useful for development where orthogonal schema changes may happen on separate branches. + + Args: + cur (Cursor) + current_version (int): The current version of the schema + delta_files (list): A list of deltas that have already been applied + upgraded (bool): Whether the current version was generated by having + applied deltas or from full schema file. If `True` the function + will never apply delta files for the given `current_version`, since + the current_version wasn't generated by applying those delta files. """ - if user_version > SCHEMA_VERSION: + if current_version > SCHEMA_VERSION: raise ValueError( "Cannot use this database as it is too " + "new for the server to understand" ) - for v in range(user_version, SCHEMA_VERSION + 1): - delta_dir = os.path.join(dir_path, "schema", "delta", v) - directory_entries = os.listdir(delta_dir) + start_ver = current_version + if not upgraded: + start_ver += 1 + + for v in range(start_ver, SCHEMA_VERSION + 1): + logger.debug("Upgrading schema to v%d", v) + + delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) + + try: + directory_entries = os.listdir(delta_dir) + except OSError: + logger.exception("Could not open delta dir for version %d", v) + raise + directory_entries.sort() for file_name in directory_entries: - relative_path = os.path.join(v, file_name) + relative_path = os.path.join(str(v), file_name) if relative_path in delta_files: continue @@ -672,17 +697,19 @@ def _upgrade_existing_database(cur, user_version, delta_files): ) root_name, ext = os.path.splitext(file_name) if ext == ".py": - module_name = "synapse.storage.schema.v%d_%s" % ( + module_name = "synapse.storage.v%d_%s" % ( v, root_name ) with open(absolute_path) as schema_file: module = imp.load_source( module_name, absolute_path, schema_file ) + logger.debug("Running script %s", relative_path) module.run_upgrade(cur) elif ext == ".sql": with open(absolute_path) as schema_file: delta_schema = schema_file.read() + logger.debug("Applying schema %s", relative_path) cur.executescript(delta_schema) else: # Not a valid delta file. @@ -695,32 +722,70 @@ def _upgrade_existing_database(cur, user_version, delta_files): # Mark as done. cur.execute( - "INSERT INTO schema_version (version, file)" + "INSERT INTO schema_deltas (version, file)" " VALUES (?,?)", (v, relative_path) ) + cur.execute( + "INSERT INTO schema_version (version, upgraded)" + " VALUES (?,?)", + (v, True) + ) + -def get_schema_state(txn): - sql = ( - "SELECT MAX(version), file FROM schema_version" - " WHERE version = (SELECT MAX(version) FROM schema_version)" +def get_or_create_schema_state(txn): + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", ) + create_schema = read_schema(schema_path) + txn.executescript(create_schema) - try: - txn.execute(sql) + txn.execute("SELECT version, upgraded FROM schema_version") + row = txn.fetchone() + current_version = int(row[0]) if row else None + upgraded = bool(row[1]) if row else None + + if current_version: + txn.execute( + "SELECT file FROM schema_deltas WHERE version >= ?", + (current_version,) + ) res = txn.fetchall() + return current_version, txn.fetchall(), upgraded - if res: - current_verison = max(r[0] for r in res) - applied_delta = [r[1] for r in res] + return None - return current_verison, applied_delta - except sqlite3.OperationalError: - txn.execute("PRAGMA user_version") - row = txn.fetchone() - if row and row[0]: - # FIXME: We need to create schema_version table! - return row[0], [] - return None +def prepare_sqlite3_database(db_conn): + """This function should be called before `prepare_database` on sqlite3 + databases. + + Since we changed the way we store the current schema version and handle + updates to schemas, we need a way to upgrade from the old method to the + new. This only affects sqlite databases since they were the only ones + supported at the time. + """ + with db_conn: + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + create_schema = read_schema(schema_path) + db_conn.executescript(create_schema) + + c = db_conn.execute("SELECT * FROM schema_version") + rows = c.fetchall() + c.close() + + if not rows: + c = db_conn.execute("PRAGMA user_version") + row = c.fetchone() + c.close() + + if row and row[0]: + ver = row[0] + db_conn.execute( + "INSERT INTO schema_version (version, upgraded)" + " VALUES (?,?)", + (row[0], False) + ) diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql deleted file mode 100644 index e491ad5aec..0000000000 --- a/synapse/storage/schema/application_services.sql +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS application_services( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT, - token TEXT, - hs_token TEXT, - sender TEXT, - UNIQUE(token) ON CONFLICT ROLLBACK -); - -CREATE TABLE IF NOT EXISTS application_services_regex( - id INTEGER PRIMARY KEY AUTOINCREMENT, - as_id INTEGER NOT NULL, - namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex TEXT, - FOREIGN KEY(as_id) REFERENCES application_services(id) -); - - - diff --git a/synapse/storage/schema/current/11/event_edges.sql b/synapse/storage/schema/current/11/event_edges.sql new file mode 100644 index 0000000000..1e766d6db2 --- /dev/null +++ b/synapse/storage/schema/current/11/event_edges.sql @@ -0,0 +1,89 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_forward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_backward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_edges( + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + is_state INTEGER NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state) +); + +CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); +CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); + + +CREATE TABLE IF NOT EXISTS room_depth( + room_id TEXT NOT NULL, + min_depth INTEGER NOT NULL, + CONSTRAINT uniqueness UNIQUE (room_id) +); + +CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); + + +create TABLE IF NOT EXISTS event_destinations( + event_id TEXT NOT NULL, + destination TEXT NOT NULL, + delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered + CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); + + +CREATE TABLE IF NOT EXISTS state_forward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( + room_id, type, state_key +); +CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_auth( + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id) +); + +CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); +CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); \ No newline at end of file diff --git a/synapse/storage/schema/current/11/event_signatures.sql b/synapse/storage/schema/current/11/event_signatures.sql new file mode 100644 index 0000000000..c28c39c48a --- /dev/null +++ b/synapse/storage/schema/current/11/event_signatures.sql @@ -0,0 +1,65 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_content_hashes ( + event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_reference_hashes ( + event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes ( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_signatures ( + event_id TEXT, + signature_name TEXT, + key_id TEXT, + signature BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id) +); + +CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_edge_hashes( + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE ( + event_id, prev_event_id, algorithm + ) +); + +CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes( + event_id +); diff --git a/synapse/storage/schema/current/11/im.sql b/synapse/storage/schema/current/11/im.sql new file mode 100644 index 0000000000..dd00c1cd2f --- /dev/null +++ b/synapse/storage/schema/current/11/im.sql @@ -0,0 +1,125 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS events( + stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, + topological_ordering INTEGER NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, + content TEXT NOT NULL, + unrecognized_keys TEXT, + processed BOOL NOT NULL, + outlier BOOL NOT NULL, + depth INTEGER DEFAULT 0 NOT NULL, + CONSTRAINT ev_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); +CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); +CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); +CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); + + +CREATE TABLE IF NOT EXISTS event_json( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + internal_metadata NOT NULL, + json BLOB NOT NULL, + CONSTRAINT ev_j_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); +CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); + + +CREATE TABLE IF NOT EXISTS state_events( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT +); + +CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id); +CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); +CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); +CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); + + +CREATE TABLE IF NOT EXISTS current_state_events( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); +CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); +CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); + +CREATE TABLE IF NOT EXISTS room_memberships( + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); +CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); + +CREATE TABLE IF NOT EXISTS feedback( + event_id TEXT NOT NULL, + feedback_type TEXT, + target_event_id TEXT, + sender TEXT, + room_id TEXT +); + +CREATE TABLE IF NOT EXISTS topics( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + topic TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); +CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); + +CREATE TABLE IF NOT EXISTS room_names( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + name TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); +CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); + +CREATE TABLE IF NOT EXISTS rooms( + room_id TEXT PRIMARY KEY NOT NULL, + is_public INTEGER, + creator TEXT +); + +CREATE TABLE IF NOT EXISTS room_hosts( + room_id TEXT NOT NULL, + host TEXT NOT NULL, + CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE +); + +CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/current/11/keys.sql b/synapse/storage/schema/current/11/keys.sql new file mode 100644 index 0000000000..a9e0a4fe0d --- /dev/null +++ b/synapse/storage/schema/current/11/keys.sql @@ -0,0 +1,31 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS server_tls_certificates( + server_name TEXT, -- Server name. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. + ts_added_ms INTEGER, -- When the certifcate was added. + tls_certificate BLOB, -- DER encoded x509 certificate. + CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) +); + +CREATE TABLE IF NOT EXISTS server_signature_keys( + server_name TEXT, -- Server name. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. + ts_added_ms INTEGER, -- When the key was added. + verify_key BLOB, -- NACL verification key. + CONSTRAINT uniqueness UNIQUE (server_name, key_id) +); diff --git a/synapse/storage/schema/current/11/media_repository.sql b/synapse/storage/schema/current/11/media_repository.sql new file mode 100644 index 0000000000..afdf48cbfb --- /dev/null +++ b/synapse/storage/schema/current/11/media_repository.sql @@ -0,0 +1,68 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS local_media_repository ( + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. + media_length INTEGER, -- Length of the media in bytes. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. + CONSTRAINT uniqueness UNIQUE (media_id) +); + +CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + CONSTRAINT uniqueness UNIQUE ( + media_id, thumbnail_width, thumbnail_height, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + +CREATE TABLE IF NOT EXISTS remote_media_cache ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + media_length INTEGER, -- Length of the media in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE (media_origin, media_id) +); + +CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE ( + media_origin, media_id, thumbnail_width, thumbnail_height, + thumbnail_type, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); diff --git a/synapse/storage/schema/current/11/presence.sql b/synapse/storage/schema/current/11/presence.sql new file mode 100644 index 0000000000..f9f8db9697 --- /dev/null +++ b/synapse/storage/schema/current/11/presence.sql @@ -0,0 +1,38 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS presence( + user_id INTEGER NOT NULL, + state INTEGER, + status_msg TEXT, + mtime INTEGER, -- miliseconds since last state change + FOREIGN KEY(user_id) REFERENCES users(id) +); + +-- For each of /my/ users which possibly-remote users are allowed to see their +-- presence state +CREATE TABLE IF NOT EXISTS presence_allow_inbound( + observed_user_id INTEGER NOT NULL, + observer_user_id TEXT, -- a UserID, + FOREIGN KEY(observed_user_id) REFERENCES users(id) +); + +-- For each of /my/ users (watcher), which possibly-remote users are they +-- watching? +CREATE TABLE IF NOT EXISTS presence_list( + user_id INTEGER NOT NULL, + observed_user_id TEXT, -- a UserID, + accepted BOOLEAN, + FOREIGN KEY(user_id) REFERENCES users(id) +); diff --git a/synapse/storage/schema/current/11/profiles.sql b/synapse/storage/schema/current/11/profiles.sql new file mode 100644 index 0000000000..f06a528b4d --- /dev/null +++ b/synapse/storage/schema/current/11/profiles.sql @@ -0,0 +1,20 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS profiles( + user_id INTEGER NOT NULL, + displayname TEXT, + avatar_url TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); diff --git a/synapse/storage/schema/current/11/redactions.sql b/synapse/storage/schema/current/11/redactions.sql new file mode 100644 index 0000000000..5011d95db8 --- /dev/null +++ b/synapse/storage/schema/current/11/redactions.sql @@ -0,0 +1,22 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS redactions ( + event_id TEXT NOT NULL, + redacts TEXT NOT NULL, + CONSTRAINT ev_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); +CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/current/11/room_aliases.sql b/synapse/storage/schema/current/11/room_aliases.sql new file mode 100644 index 0000000000..0d2df01603 --- /dev/null +++ b/synapse/storage/schema/current/11/room_aliases.sql @@ -0,0 +1,27 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS room_aliases( + room_alias TEXT NOT NULL, + room_id TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS room_alias_servers( + room_alias TEXT NOT NULL, + server TEXT NOT NULL +); + + + diff --git a/synapse/storage/schema/current/11/state.sql b/synapse/storage/schema/current/11/state.sql new file mode 100644 index 0000000000..1fe8f1e430 --- /dev/null +++ b/synapse/storage/schema/current/11/state.sql @@ -0,0 +1,47 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS state_groups( + id INTEGER PRIMARY KEY, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS state_groups_state( + state_group INTEGER NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS event_to_state_groups( + event_id TEXT NOT NULL, + state_group INTEGER NOT NULL, + CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); + +CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state( + state_group +); +CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state( + room_id, type, state_key +); + +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups( + event_id +); \ No newline at end of file diff --git a/synapse/storage/schema/current/11/transactions.sql b/synapse/storage/schema/current/11/transactions.sql new file mode 100644 index 0000000000..2d30f99b06 --- /dev/null +++ b/synapse/storage/schema/current/11/transactions.sql @@ -0,0 +1,68 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Stores what transaction ids we have received and what our response was +CREATE TABLE IF NOT EXISTS received_transactions( + transaction_id TEXT, + origin TEXT, + ts INTEGER, + response_code INTEGER, + response_json TEXT, + has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + CONSTRAINT uniquesss UNIQUE (transaction_id, origin) ON CONFLICT REPLACE +); + +CREATE UNIQUE INDEX IF NOT EXISTS transactions_txid ON received_transactions(transaction_id, origin); +CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; + + +-- Stores what transactions we've sent, what their response was (if we got one) and whether we have +-- since referenced the transaction in another outgoing transaction +CREATE TABLE IF NOT EXISTS sent_transactions( + id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering + transaction_id TEXT, + destination TEXT, + response_code INTEGER DEFAULT 0, + response_json TEXT, + ts INTEGER +); + +CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( + destination +); +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +-- So that we can do an efficient look up of all transactions that have yet to be successfully +-- sent. +CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); + + +-- For sent transactions only. +CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( + transaction_id INTEGER, + destination TEXT, + pdu_id TEXT, + pdu_origin TEXT +); + +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination); + +-- To track destination health +CREATE TABLE IF NOT EXISTS destinations( + destination TEXT PRIMARY KEY, + retry_last_ts INTEGER, + retry_interval INTEGER +); diff --git a/synapse/storage/schema/current/11/users.sql b/synapse/storage/schema/current/11/users.sql new file mode 100644 index 0000000000..08ccfdac0a --- /dev/null +++ b/synapse/storage/schema/current/11/users.sql @@ -0,0 +1,45 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS users( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + password_hash TEXT, + creation_ts INTEGER, + admin BOOL DEFAULT 0 NOT NULL, + UNIQUE(name) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS access_tokens( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + device_id TEXT, + token TEXT NOT NULL, + last_used INTEGER, + FOREIGN KEY(user_id) REFERENCES users(id), + UNIQUE(token) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS user_ips ( + user TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, + user_agent TEXT NOT NULL, + last_seen INTEGER NOT NULL, + CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); + diff --git a/synapse/storage/schema/delta/11/v11.sql b/synapse/storage/schema/delta/11/v11.sql new file mode 100644 index 0000000000..313592221b --- /dev/null +++ b/synapse/storage/schema/delta/11/v11.sql @@ -0,0 +1,16 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); \ No newline at end of file diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql new file mode 100644 index 0000000000..b87ef1fe79 --- /dev/null +++ b/synapse/storage/schema/delta/12/v12.sql @@ -0,0 +1,67 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE +); + +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + profile_tag varchar(32) NOT NULL, + kind varchar(8) NOT NULL, + app_id varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + ts BIGINT NOT NULL, + lang varchar(8), + data blob, + last_token TEXT, + last_success BIGINT, + failing_since BIGINT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (app_id, pushkey) +); + +CREATE TABLE IF NOT EXISTS push_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); + +CREATE TABLE IF NOT EXISTS user_filters( + user_id TEXT, + filter_id INTEGER, + filter_json TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); + +PRAGMA user_version = 12; diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql new file mode 100644 index 0000000000..e491ad5aec --- /dev/null +++ b/synapse/storage/schema/delta/13/v13.sql @@ -0,0 +1,34 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT, + token TEXT, + hs_token TEXT, + sender TEXT, + UNIQUE(token) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id INTEGER PRIMARY KEY AUTOINCREMENT, + as_id INTEGER NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex TEXT, + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + + + diff --git a/synapse/storage/schema/delta/14/upgrade_appservice_db.py b/synapse/storage/schema/delta/14/upgrade_appservice_db.py new file mode 100644 index 0000000000..55e43c41ab --- /dev/null +++ b/synapse/storage/schema/delta/14/upgrade_appservice_db.py @@ -0,0 +1,20 @@ +import json + + +def run_upgrade(cur): + cur.execute("SELECT id, regex FROM application_services_regex") + for row in cur.fetchall(): + try: + print "checking %s..." % row[0] + json.loads(row[1]) + except ValueError: + # row isn't in json, make it so. + string_regex = row[1] + new_regex = json.dumps({ + "regex": string_regex, + "exclusive": True + }) + cur.execute( + "UPDATE application_services_regex SET regex=? WHERE id=?", + (new_regex, row[0]) + ) diff --git a/synapse/storage/schema/delta/v11.sql b/synapse/storage/schema/delta/v11.sql deleted file mode 100644 index 313592221b..0000000000 --- a/synapse/storage/schema/delta/v11.sql +++ /dev/null @@ -1,16 +0,0 @@ -/* Copyright 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); \ No newline at end of file diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql deleted file mode 100644 index b87ef1fe79..0000000000 --- a/synapse/storage/schema/delta/v12.sql +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS rejections( - event_id TEXT NOT NULL, - reason TEXT NOT NULL, - last_check TEXT NOT NULL, - CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE -); - --- Push notification endpoints that users have configured -CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - profile_tag varchar(32) NOT NULL, - kind varchar(8) NOT NULL, - app_id varchar(64) NOT NULL, - app_display_name varchar(64) NOT NULL, - device_display_name varchar(128) NOT NULL, - pushkey blob NOT NULL, - ts BIGINT NOT NULL, - lang varchar(8), - data blob, - last_token TEXT, - last_success BIGINT, - failing_since BIGINT, - FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (app_id, pushkey) -); - -CREATE TABLE IF NOT EXISTS push_rules ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, - priority_class TINYINT NOT NULL, - priority INTEGER NOT NULL DEFAULT 0, - conditions TEXT NOT NULL, - actions TEXT NOT NULL, - UNIQUE(user_name, rule_id) -); - -CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); - -CREATE TABLE IF NOT EXISTS user_filters( - user_id TEXT, - filter_id INTEGER, - filter_json TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) -); - -CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( - user_id, filter_id -); - -PRAGMA user_version = 12; diff --git a/synapse/storage/schema/delta/v13.sql b/synapse/storage/schema/delta/v13.sql deleted file mode 100644 index e491ad5aec..0000000000 --- a/synapse/storage/schema/delta/v13.sql +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS application_services( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT, - token TEXT, - hs_token TEXT, - sender TEXT, - UNIQUE(token) ON CONFLICT ROLLBACK -); - -CREATE TABLE IF NOT EXISTS application_services_regex( - id INTEGER PRIMARY KEY AUTOINCREMENT, - as_id INTEGER NOT NULL, - namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex TEXT, - FOREIGN KEY(as_id) REFERENCES application_services(id) -); - - - diff --git a/synapse/storage/schema/delta/v2.sql b/synapse/storage/schema/delta/v2.sql deleted file mode 100644 index f740f6dd5d..0000000000 --- a/synapse/storage/schema/delta/v2.sql +++ /dev/null @@ -1,168 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS events( - stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, - topological_ordering INTEGER NOT NULL, - event_id TEXT NOT NULL, - type TEXT NOT NULL, - room_id TEXT NOT NULL, - content TEXT NOT NULL, - unrecognized_keys TEXT, - processed BOOL NOT NULL, - outlier BOOL NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); -CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); -CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); -CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); - -CREATE TABLE IF NOT EXISTS state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - prev_state TEXT -); - -CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id); -CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); -CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); -CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); - - -CREATE TABLE IF NOT EXISTS current_state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); -CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); -CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); -CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); - -CREATE TABLE IF NOT EXISTS room_memberships( - event_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sender TEXT NOT NULL, - room_id TEXT NOT NULL, - membership TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); -CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); -CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); - -CREATE TABLE IF NOT EXISTS feedback( - event_id TEXT NOT NULL, - feedback_type TEXT, - target_event_id TEXT, - sender TEXT, - room_id TEXT -); - -CREATE TABLE IF NOT EXISTS topics( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - topic TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS room_names( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - name TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS rooms( - room_id TEXT PRIMARY KEY NOT NULL, - is_public INTEGER, - creator TEXT -); - -CREATE TABLE IF NOT EXISTS room_join_rules( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - join_rule TEXT NOT NULL -); -CREATE INDEX IF NOT EXISTS room_join_rules_event_id ON room_join_rules(event_id); -CREATE INDEX IF NOT EXISTS room_join_rules_room_id ON room_join_rules(room_id); - - -CREATE TABLE IF NOT EXISTS room_power_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - user_id TEXT NOT NULL, - level INTEGER NOT NULL -); -CREATE INDEX IF NOT EXISTS room_power_levels_event_id ON room_power_levels(event_id); -CREATE INDEX IF NOT EXISTS room_power_levels_room_id ON room_power_levels(room_id); -CREATE INDEX IF NOT EXISTS room_power_levels_room_user ON room_power_levels(room_id, user_id); - - -CREATE TABLE IF NOT EXISTS room_default_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - level INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_default_levels_event_id ON room_default_levels(event_id); -CREATE INDEX IF NOT EXISTS room_default_levels_room_id ON room_default_levels(room_id); - - -CREATE TABLE IF NOT EXISTS room_add_state_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - level INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_add_state_levels_event_id ON room_add_state_levels(event_id); -CREATE INDEX IF NOT EXISTS room_add_state_levels_room_id ON room_add_state_levels(room_id); - - -CREATE TABLE IF NOT EXISTS room_send_event_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - level INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_send_event_levels_event_id ON room_send_event_levels(event_id); -CREATE INDEX IF NOT EXISTS room_send_event_levels_room_id ON room_send_event_levels(room_id); - - -CREATE TABLE IF NOT EXISTS room_ops_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - ban_level INTEGER, - kick_level INTEGER -); - -CREATE INDEX IF NOT EXISTS room_ops_levels_event_id ON room_ops_levels(event_id); -CREATE INDEX IF NOT EXISTS room_ops_levels_room_id ON room_ops_levels(room_id); - - -CREATE TABLE IF NOT EXISTS room_hosts( - room_id TEXT NOT NULL, - host TEXT NOT NULL, - CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE -); - -CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); - -PRAGMA user_version = 2; diff --git a/synapse/storage/schema/delta/v3.sql b/synapse/storage/schema/delta/v3.sql deleted file mode 100644 index c67e38ff52..0000000000 --- a/synapse/storage/schema/delta/v3.sql +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -CREATE INDEX IF NOT EXISTS room_aliases_alias ON room_aliases(room_alias); -CREATE INDEX IF NOT EXISTS room_aliases_id ON room_aliases(room_id); - - -CREATE INDEX IF NOT EXISTS room_alias_servers_alias ON room_alias_servers(room_alias); - -DELETE FROM room_aliases WHERE rowid NOT IN (SELECT max(rowid) FROM room_aliases GROUP BY room_alias, room_id); - -CREATE UNIQUE INDEX IF NOT EXISTS room_aliases_uniq ON room_aliases(room_alias, room_id); - -PRAGMA user_version = 3; diff --git a/synapse/storage/schema/delta/v4.sql b/synapse/storage/schema/delta/v4.sql deleted file mode 100644 index d3807b7686..0000000000 --- a/synapse/storage/schema/delta/v4.sql +++ /dev/null @@ -1,26 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS redactions ( - event_id TEXT NOT NULL, - redacts TEXT NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); -CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); - -ALTER TABLE room_ops_levels ADD COLUMN redact_level INTEGER; - -PRAGMA user_version = 4; diff --git a/synapse/storage/schema/delta/v5.sql b/synapse/storage/schema/delta/v5.sql deleted file mode 100644 index 0874a15431..0000000000 --- a/synapse/storage/schema/delta/v5.sql +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS user_ips ( - user TEXT NOT NULL, - access_token TEXT NOT NULL, - device_id TEXT, - ip TEXT NOT NULL, - user_agent TEXT NOT NULL, - last_seen INTEGER NOT NULL, - CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); - -ALTER TABLE users ADD COLUMN admin BOOL DEFAULT 0 NOT NULL; - -PRAGMA user_version = 5; diff --git a/synapse/storage/schema/delta/v6.sql b/synapse/storage/schema/delta/v6.sql deleted file mode 100644 index a9e0a4fe0d..0000000000 --- a/synapse/storage/schema/delta/v6.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name TEXT, -- Server name. - fingerprint TEXT, -- Certificate fingerprint. - from_server TEXT, -- Which key server the certificate was fetched from. - ts_added_ms INTEGER, -- When the certifcate was added. - tls_certificate BLOB, -- DER encoded x509 certificate. - CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) -); - -CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name TEXT, -- Server name. - key_id TEXT, -- Key version. - from_server TEXT, -- Which key server the key was fetched form. - ts_added_ms INTEGER, -- When the key was added. - verify_key BLOB, -- NACL verification key. - CONSTRAINT uniqueness UNIQUE (server_name, key_id) -); diff --git a/synapse/storage/schema/delta/v8.sql b/synapse/storage/schema/delta/v8.sql deleted file mode 100644 index 1e9f8b18cb..0000000000 --- a/synapse/storage/schema/delta/v8.sql +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - CREATE TABLE IF NOT EXISTS event_signatures_2 ( - event_id TEXT, - signature_name TEXT, - key_id TEXT, - signature BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id) -); - -INSERT INTO event_signatures_2 (event_id, signature_name, key_id, signature) -SELECT event_id, signature_name, key_id, signature FROM event_signatures; - -DROP TABLE event_signatures; -ALTER TABLE event_signatures_2 RENAME TO event_signatures; - -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( - event_id -); - -PRAGMA user_version = 8; \ No newline at end of file diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql deleted file mode 100644 index 455d51a70c..0000000000 --- a/synapse/storage/schema/delta/v9.sql +++ /dev/null @@ -1,79 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- To track destination health -CREATE TABLE IF NOT EXISTS destinations( - destination TEXT PRIMARY KEY, - retry_last_ts INTEGER, - retry_interval INTEGER -); - - -CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id TEXT, -- The id used to refer to the media. - media_type TEXT, -- The MIME-type of the media. - media_length INTEGER, -- Length of the media in bytes. - created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. - user_id TEXT, -- The user who uploaded the file. - CONSTRAINT uniqueness UNIQUE (media_id) -); - -CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id TEXT, -- The id used to refer to the media. - thumbnail_width INTEGER, -- The width of the thumbnail in pixels. - thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type TEXT, -- The MIME-type of the thumbnail. - thumbnail_method TEXT, -- The method used to make the thumbnail. - thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - CONSTRAINT uniqueness UNIQUE ( - media_id, thumbnail_width, thumbnail_height, thumbnail_type - ) -); - -CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id - ON local_media_repository_thumbnails (media_id); - -CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media on that server. - media_type TEXT, -- The MIME-type of the media. - created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. - media_length INTEGER, -- Length of the media in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE (media_origin, media_id) -); - -CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media. - thumbnail_width INTEGER, -- The width of the thumbnail in pixels. - thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method TEXT, -- The method used to make the thumbnail - thumbnail_type TEXT, -- The MIME-type of the thumbnail. - thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE ( - media_origin, media_id, thumbnail_width, thumbnail_height, - thumbnail_type, thumbnail_type - ) -); - -CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id - ON local_media_repository_thumbnails (media_id); - - -PRAGMA user_version = 9; diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql deleted file mode 100644 index 1e766d6db2..0000000000 --- a/synapse/storage/schema/event_edges.sql +++ /dev/null @@ -1,89 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); - - -CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); - - -CREATE TABLE IF NOT EXISTS event_edges( - event_id TEXT NOT NULL, - prev_event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - is_state INTEGER NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state) -); - -CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); -CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); - - -CREATE TABLE IF NOT EXISTS room_depth( - room_id TEXT NOT NULL, - min_depth INTEGER NOT NULL, - CONSTRAINT uniqueness UNIQUE (room_id) -); - -CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); - - -create TABLE IF NOT EXISTS event_destinations( - event_id TEXT NOT NULL, - destination TEXT NOT NULL, - delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered - CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); - - -CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( - room_id, type, state_key -); -CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); - - -CREATE TABLE IF NOT EXISTS event_auth( - event_id TEXT NOT NULL, - auth_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id) -); - -CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); \ No newline at end of file diff --git a/synapse/storage/schema/event_signatures.sql b/synapse/storage/schema/event_signatures.sql deleted file mode 100644 index c28c39c48a..0000000000 --- a/synapse/storage/schema/event_signatures.sql +++ /dev/null @@ -1,65 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, algorithm) -); - -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes( - event_id -); - - -CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, algorithm) -); - -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes ( - event_id -); - - -CREATE TABLE IF NOT EXISTS event_signatures ( - event_id TEXT, - signature_name TEXT, - key_id TEXT, - signature BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id) -); - -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( - event_id -); - - -CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id TEXT, - prev_event_id TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE ( - event_id, prev_event_id, algorithm - ) -); - -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes( - event_id -); diff --git a/synapse/storage/schema/filtering.sql b/synapse/storage/schema/filtering.sql deleted file mode 100644 index beb39ca201..0000000000 --- a/synapse/storage/schema/filtering.sql +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS user_filters( - user_id TEXT, - filter_id INTEGER, - filter_json TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) -); - -CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( - user_id, filter_id -); diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql deleted file mode 100644 index dd00c1cd2f..0000000000 --- a/synapse/storage/schema/im.sql +++ /dev/null @@ -1,125 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS events( - stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, - topological_ordering INTEGER NOT NULL, - event_id TEXT NOT NULL, - type TEXT NOT NULL, - room_id TEXT NOT NULL, - content TEXT NOT NULL, - unrecognized_keys TEXT, - processed BOOL NOT NULL, - outlier BOOL NOT NULL, - depth INTEGER DEFAULT 0 NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); -CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); -CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); -CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); - - -CREATE TABLE IF NOT EXISTS event_json( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - internal_metadata NOT NULL, - json BLOB NOT NULL, - CONSTRAINT ev_j_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); -CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); - - -CREATE TABLE IF NOT EXISTS state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - prev_state TEXT -); - -CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id); -CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); -CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); -CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); - - -CREATE TABLE IF NOT EXISTS current_state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); -CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); -CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); -CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); - -CREATE TABLE IF NOT EXISTS room_memberships( - event_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sender TEXT NOT NULL, - room_id TEXT NOT NULL, - membership TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); -CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); -CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); - -CREATE TABLE IF NOT EXISTS feedback( - event_id TEXT NOT NULL, - feedback_type TEXT, - target_event_id TEXT, - sender TEXT, - room_id TEXT -); - -CREATE TABLE IF NOT EXISTS topics( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - topic TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); -CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); - -CREATE TABLE IF NOT EXISTS room_names( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - name TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); -CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); - -CREATE TABLE IF NOT EXISTS rooms( - room_id TEXT PRIMARY KEY NOT NULL, - is_public INTEGER, - creator TEXT -); - -CREATE TABLE IF NOT EXISTS room_hosts( - room_id TEXT NOT NULL, - host TEXT NOT NULL, - CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE -); - -CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/keys.sql b/synapse/storage/schema/keys.sql deleted file mode 100644 index a9e0a4fe0d..0000000000 --- a/synapse/storage/schema/keys.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name TEXT, -- Server name. - fingerprint TEXT, -- Certificate fingerprint. - from_server TEXT, -- Which key server the certificate was fetched from. - ts_added_ms INTEGER, -- When the certifcate was added. - tls_certificate BLOB, -- DER encoded x509 certificate. - CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) -); - -CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name TEXT, -- Server name. - key_id TEXT, -- Key version. - from_server TEXT, -- Which key server the key was fetched form. - ts_added_ms INTEGER, -- When the key was added. - verify_key BLOB, -- NACL verification key. - CONSTRAINT uniqueness UNIQUE (server_name, key_id) -); diff --git a/synapse/storage/schema/media_repository.sql b/synapse/storage/schema/media_repository.sql deleted file mode 100644 index afdf48cbfb..0000000000 --- a/synapse/storage/schema/media_repository.sql +++ /dev/null @@ -1,68 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id TEXT, -- The id used to refer to the media. - media_type TEXT, -- The MIME-type of the media. - media_length INTEGER, -- Length of the media in bytes. - created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. - user_id TEXT, -- The user who uploaded the file. - CONSTRAINT uniqueness UNIQUE (media_id) -); - -CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id TEXT, -- The id used to refer to the media. - thumbnail_width INTEGER, -- The width of the thumbnail in pixels. - thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type TEXT, -- The MIME-type of the thumbnail. - thumbnail_method TEXT, -- The method used to make the thumbnail. - thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - CONSTRAINT uniqueness UNIQUE ( - media_id, thumbnail_width, thumbnail_height, thumbnail_type - ) -); - -CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id - ON local_media_repository_thumbnails (media_id); - -CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media on that server. - media_type TEXT, -- The MIME-type of the media. - created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. - media_length INTEGER, -- Length of the media in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE (media_origin, media_id) -); - -CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media. - thumbnail_width INTEGER, -- The width of the thumbnail in pixels. - thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method TEXT, -- The method used to make the thumbnail - thumbnail_type TEXT, -- The MIME-type of the thumbnail. - thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE ( - media_origin, media_id, thumbnail_width, thumbnail_height, - thumbnail_type, thumbnail_type - ) -); - -CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id - ON local_media_repository_thumbnails (media_id); diff --git a/synapse/storage/schema/presence.sql b/synapse/storage/schema/presence.sql deleted file mode 100644 index f9f8db9697..0000000000 --- a/synapse/storage/schema/presence.sql +++ /dev/null @@ -1,38 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS presence( - user_id INTEGER NOT NULL, - state INTEGER, - status_msg TEXT, - mtime INTEGER, -- miliseconds since last state change - FOREIGN KEY(user_id) REFERENCES users(id) -); - --- For each of /my/ users which possibly-remote users are allowed to see their --- presence state -CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id INTEGER NOT NULL, - observer_user_id TEXT, -- a UserID, - FOREIGN KEY(observed_user_id) REFERENCES users(id) -); - --- For each of /my/ users (watcher), which possibly-remote users are they --- watching? -CREATE TABLE IF NOT EXISTS presence_list( - user_id INTEGER NOT NULL, - observed_user_id TEXT, -- a UserID, - accepted BOOLEAN, - FOREIGN KEY(user_id) REFERENCES users(id) -); diff --git a/synapse/storage/schema/profiles.sql b/synapse/storage/schema/profiles.sql deleted file mode 100644 index f06a528b4d..0000000000 --- a/synapse/storage/schema/profiles.sql +++ /dev/null @@ -1,20 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS profiles( - user_id INTEGER NOT NULL, - displayname TEXT, - avatar_url TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) -); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql deleted file mode 100644 index 3735b11547..0000000000 --- a/synapse/storage/schema/pusher.sql +++ /dev/null @@ -1,46 +0,0 @@ -/* Copyright 2014 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- Push notification endpoints that users have configured -CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - profile_tag varchar(32) NOT NULL, - kind varchar(8) NOT NULL, - app_id varchar(64) NOT NULL, - app_display_name varchar(64) NOT NULL, - device_display_name varchar(128) NOT NULL, - pushkey blob NOT NULL, - ts BIGINT NOT NULL, - lang varchar(8), - data blob, - last_token TEXT, - last_success BIGINT, - failing_since BIGINT, - FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (app_id, pushkey) -); - -CREATE TABLE IF NOT EXISTS push_rules ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, - priority_class TINYINT NOT NULL, - priority INTEGER NOT NULL DEFAULT 0, - conditions TEXT NOT NULL, - actions TEXT NOT NULL, - UNIQUE(user_name, rule_id) -); - -CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); diff --git a/synapse/storage/schema/redactions.sql b/synapse/storage/schema/redactions.sql deleted file mode 100644 index 5011d95db8..0000000000 --- a/synapse/storage/schema/redactions.sql +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS redactions ( - event_id TEXT NOT NULL, - redacts TEXT NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); -CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/rejections.sql b/synapse/storage/schema/rejections.sql deleted file mode 100644 index bd2a8b1bb5..0000000000 --- a/synapse/storage/schema/rejections.sql +++ /dev/null @@ -1,21 +0,0 @@ -/* Copyright 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS rejections( - event_id TEXT NOT NULL, - reason TEXT NOT NULL, - last_check TEXT NOT NULL, - CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE -); diff --git a/synapse/storage/schema/room_aliases.sql b/synapse/storage/schema/room_aliases.sql deleted file mode 100644 index 0d2df01603..0000000000 --- a/synapse/storage/schema/room_aliases.sql +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS room_aliases( - room_alias TEXT NOT NULL, - room_id TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias TEXT NOT NULL, - server TEXT NOT NULL -); - - - diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql new file mode 100644 index 0000000000..83a8c7b7ce --- /dev/null +++ b/synapse/storage/schema/schema_version.sql @@ -0,0 +1,29 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS schema_version( + Lock char(1) NOT NULL DEFAULT 'X', -- Makes sure this table only has one row. + version INTEGER NOT NULL, + upgraded BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. + CONSTRAINT schema_version_lock CHECK (Lock='X') ON CONFLICT REPLACE +); + +CREATE TABLE IF NOT EXISTS schema_deltas( + version INTEGER NOT NULL, + file TEXT NOT NULL, + CONSTRAINT schema_deltas_ver_file UNIQUE (version, file) ON CONFLICT IGNORE +); + +CREATE INDEX IF NOT EXISTS schema_deltas_ver ON schema_deltas(version); diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/state.sql deleted file mode 100644 index 1fe8f1e430..0000000000 --- a/synapse/storage/schema/state.sql +++ /dev/null @@ -1,47 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS state_groups( - id INTEGER PRIMARY KEY, - room_id TEXT NOT NULL, - event_id TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS state_groups_state( - state_group INTEGER NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - event_id TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id TEXT NOT NULL, - state_group INTEGER NOT NULL, - CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); - -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state( - state_group -); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state( - room_id, type, state_key -); - -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups( - event_id -); \ No newline at end of file diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql deleted file mode 100644 index 2d30f99b06..0000000000 --- a/synapse/storage/schema/transactions.sql +++ /dev/null @@ -1,68 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- Stores what transaction ids we have received and what our response was -CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id TEXT, - origin TEXT, - ts INTEGER, - response_code INTEGER, - response_json TEXT, - has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx - CONSTRAINT uniquesss UNIQUE (transaction_id, origin) ON CONFLICT REPLACE -); - -CREATE UNIQUE INDEX IF NOT EXISTS transactions_txid ON received_transactions(transaction_id, origin); -CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; - - --- Stores what transactions we've sent, what their response was (if we got one) and whether we have --- since referenced the transaction in another outgoing transaction -CREATE TABLE IF NOT EXISTS sent_transactions( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering - transaction_id TEXT, - destination TEXT, - response_code INTEGER DEFAULT 0, - response_json TEXT, - ts INTEGER -); - -CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( - destination -); -CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); --- So that we can do an efficient look up of all transactions that have yet to be successfully --- sent. -CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); - - --- For sent transactions only. -CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( - transaction_id INTEGER, - destination TEXT, - pdu_id TEXT, - pdu_origin TEXT -); - -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination); - --- To track destination health -CREATE TABLE IF NOT EXISTS destinations( - destination TEXT PRIMARY KEY, - retry_last_ts INTEGER, - retry_interval INTEGER -); diff --git a/synapse/storage/schema/users.sql b/synapse/storage/schema/users.sql deleted file mode 100644 index 08ccfdac0a..0000000000 --- a/synapse/storage/schema/users.sql +++ /dev/null @@ -1,45 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS users( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT, - password_hash TEXT, - creation_ts INTEGER, - admin BOOL DEFAULT 0 NOT NULL, - UNIQUE(name) ON CONFLICT ROLLBACK -); - -CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id INTEGER NOT NULL, - device_id TEXT, - token TEXT NOT NULL, - last_used INTEGER, - FOREIGN KEY(user_id) REFERENCES users(id), - UNIQUE(token) ON CONFLICT ROLLBACK -); - -CREATE TABLE IF NOT EXISTS user_ips ( - user TEXT NOT NULL, - access_token TEXT NOT NULL, - device_id TEXT, - ip TEXT NOT NULL, - user_agent TEXT NOT NULL, - last_seen INTEGER NOT NULL, - CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); - -- cgit 1.4.1 From 811355ccd0b1a22560ba0af57fd3ad1e07ad283f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 13:11:01 +0000 Subject: Add some docs and remove unused variables --- synapse/storage/__init__.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 07ccc4e2ee..565abe81d9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -613,7 +613,7 @@ def _setup_new_database(cur): valid_dirs.append((ver, abs_path)) if not valid_dirs: - raise RuntimeError("Could not find a suitable current.sql") + raise PrepareDatabaseException("Could not find a suitable current.sql") max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0]) @@ -684,7 +684,9 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded): directory_entries = os.listdir(delta_dir) except OSError: logger.exception("Could not open delta dir for version %d", v) - raise + raise UpgradeDatabaseException( + "Could not open delta dir for version %d" % (v,) + ) directory_entries.sort() for file_name in directory_entries: @@ -697,18 +699,20 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded): ) root_name, ext = os.path.splitext(file_name) if ext == ".py": + # This is a python upgrade module. We need to import into some + # package and then execute its `run_upgrade` function. module_name = "synapse.storage.v%d_%s" % ( v, root_name ) - with open(absolute_path) as schema_file: + with open(absolute_path) as python_file: module = imp.load_source( - module_name, absolute_path, schema_file + module_name, absolute_path, python_file ) logger.debug("Running script %s", relative_path) module.run_upgrade(cur) elif ext == ".sql": - with open(absolute_path) as schema_file: - delta_schema = schema_file.read() + # A plain old .sql file, just read and execute it + delta_schema = read_schema(absolute_path) logger.debug("Applying schema %s", relative_path) cur.executescript(delta_schema) else: @@ -751,7 +755,6 @@ def get_or_create_schema_state(txn): "SELECT file FROM schema_deltas WHERE version >= ?", (current_version,) ) - res = txn.fetchall() return current_version, txn.fetchall(), upgraded return None @@ -783,7 +786,6 @@ def prepare_sqlite3_database(db_conn): c.close() if row and row[0]: - ver = row[0] db_conn.execute( "INSERT INTO schema_version (version, upgraded)" " VALUES (?,?)", -- cgit 1.4.1 From c3530c3fb3f374bcb93097f23d311db979912d92 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 13:34:11 +0000 Subject: More docs. Rename 'schema/current' to 'schema/full_schemas' --- synapse/storage/__init__.py | 30 +++-- synapse/storage/schema/current/11/event_edges.sql | 89 --------------- .../storage/schema/current/11/event_signatures.sql | 65 ----------- synapse/storage/schema/current/11/im.sql | 125 --------------------- synapse/storage/schema/current/11/keys.sql | 31 ----- .../storage/schema/current/11/media_repository.sql | 68 ----------- synapse/storage/schema/current/11/presence.sql | 38 ------- synapse/storage/schema/current/11/profiles.sql | 20 ---- synapse/storage/schema/current/11/redactions.sql | 22 ---- synapse/storage/schema/current/11/room_aliases.sql | 27 ----- synapse/storage/schema/current/11/state.sql | 47 -------- synapse/storage/schema/current/11/transactions.sql | 68 ----------- synapse/storage/schema/current/11/users.sql | 45 -------- 13 files changed, 23 insertions(+), 652 deletions(-) delete mode 100644 synapse/storage/schema/current/11/event_edges.sql delete mode 100644 synapse/storage/schema/current/11/event_signatures.sql delete mode 100644 synapse/storage/schema/current/11/im.sql delete mode 100644 synapse/storage/schema/current/11/keys.sql delete mode 100644 synapse/storage/schema/current/11/media_repository.sql delete mode 100644 synapse/storage/schema/current/11/presence.sql delete mode 100644 synapse/storage/schema/current/11/profiles.sql delete mode 100644 synapse/storage/schema/current/11/redactions.sql delete mode 100644 synapse/storage/schema/current/11/room_aliases.sql delete mode 100644 synapse/storage/schema/current/11/state.sql delete mode 100644 synapse/storage/schema/current/11/transactions.sql delete mode 100644 synapse/storage/schema/current/11/users.sql (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 565abe81d9..a22a0e2f39 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -55,8 +55,8 @@ import re logger = logging.getLogger(__name__) -# Remember to update this number every time an incompatible change is made to -# database schema files, so the users will be informed on server restarts. +# Remember to update this number every time an change is made to database +# schema files, so the users will be informed on server restarts. SCHEMA_VERSION = 14 dir_path = os.path.abspath(os.path.dirname(__file__)) @@ -584,7 +584,7 @@ def prepare_database(db_conn): or upgrade from an older schema version. """ cur = db_conn.cursor() - version_info = get_or_create_schema_state(cur) + version_info = _get_or_create_schema_state(cur) if version_info: user_version, delta_files, upgraded = version_info @@ -599,7 +599,17 @@ def prepare_database(db_conn): def _setup_new_database(cur): - current_dir = os.path.join(dir_path, "schema", "current") + """Sets up the database by finding a base set of "full schemas" and then + applying any necessary deltas. + + The "full_schemas" directory has subdirectories named after versions. This + function searches for the highest version less than or equal to + `SCHEMA_VERSION` and excutes all .sql files in that directory. + + The function will then apply all deltas for all versions after the base + version. + """ + current_dir = os.path.join(dir_path, "schema", "full_schemas") directory_entries = os.listdir(current_dir) valid_dirs = [] @@ -609,11 +619,15 @@ def _setup_new_database(cur): abs_path = os.path.join(current_dir, filename) if match and os.path.isdir(abs_path): ver = int(match.group(0)) - if ver < SCHEMA_VERSION: + if ver <= SCHEMA_VERSION: valid_dirs.append((ver, abs_path)) + else: + logger.warn("Unexpected entry in 'full_schemas': %s", filename) if not valid_dirs: - raise PrepareDatabaseException("Could not find a suitable current.sql") + raise PrepareDatabaseException( + "Could not find a suitable base set of full schemas" + ) max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0]) @@ -655,6 +669,8 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded): even if there has been no version bump. This is useful for development where orthogonal schema changes may happen on separate branches. + This is a no-op of current_version == SCHEMA_VERSION. + Args: cur (Cursor) current_version (int): The current version of the schema @@ -738,7 +754,7 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded): ) -def get_or_create_schema_state(txn): +def _get_or_create_schema_state(txn): schema_path = os.path.join( dir_path, "schema", "schema_version.sql", ) diff --git a/synapse/storage/schema/current/11/event_edges.sql b/synapse/storage/schema/current/11/event_edges.sql deleted file mode 100644 index 1e766d6db2..0000000000 --- a/synapse/storage/schema/current/11/event_edges.sql +++ /dev/null @@ -1,89 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); - - -CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); - - -CREATE TABLE IF NOT EXISTS event_edges( - event_id TEXT NOT NULL, - prev_event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - is_state INTEGER NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state) -); - -CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); -CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); - - -CREATE TABLE IF NOT EXISTS room_depth( - room_id TEXT NOT NULL, - min_depth INTEGER NOT NULL, - CONSTRAINT uniqueness UNIQUE (room_id) -); - -CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); - - -create TABLE IF NOT EXISTS event_destinations( - event_id TEXT NOT NULL, - destination TEXT NOT NULL, - delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered - CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); - - -CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( - room_id, type, state_key -); -CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); - - -CREATE TABLE IF NOT EXISTS event_auth( - event_id TEXT NOT NULL, - auth_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id) -); - -CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); \ No newline at end of file diff --git a/synapse/storage/schema/current/11/event_signatures.sql b/synapse/storage/schema/current/11/event_signatures.sql deleted file mode 100644 index c28c39c48a..0000000000 --- a/synapse/storage/schema/current/11/event_signatures.sql +++ /dev/null @@ -1,65 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, algorithm) -); - -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes( - event_id -); - - -CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, algorithm) -); - -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes ( - event_id -); - - -CREATE TABLE IF NOT EXISTS event_signatures ( - event_id TEXT, - signature_name TEXT, - key_id TEXT, - signature BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id) -); - -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( - event_id -); - - -CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id TEXT, - prev_event_id TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE ( - event_id, prev_event_id, algorithm - ) -); - -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes( - event_id -); diff --git a/synapse/storage/schema/current/11/im.sql b/synapse/storage/schema/current/11/im.sql deleted file mode 100644 index dd00c1cd2f..0000000000 --- a/synapse/storage/schema/current/11/im.sql +++ /dev/null @@ -1,125 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS events( - stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, - topological_ordering INTEGER NOT NULL, - event_id TEXT NOT NULL, - type TEXT NOT NULL, - room_id TEXT NOT NULL, - content TEXT NOT NULL, - unrecognized_keys TEXT, - processed BOOL NOT NULL, - outlier BOOL NOT NULL, - depth INTEGER DEFAULT 0 NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); -CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); -CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); -CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); - - -CREATE TABLE IF NOT EXISTS event_json( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - internal_metadata NOT NULL, - json BLOB NOT NULL, - CONSTRAINT ev_j_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); -CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); - - -CREATE TABLE IF NOT EXISTS state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - prev_state TEXT -); - -CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id); -CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); -CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); -CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); - - -CREATE TABLE IF NOT EXISTS current_state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); -CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); -CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); -CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); - -CREATE TABLE IF NOT EXISTS room_memberships( - event_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sender TEXT NOT NULL, - room_id TEXT NOT NULL, - membership TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); -CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); -CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); - -CREATE TABLE IF NOT EXISTS feedback( - event_id TEXT NOT NULL, - feedback_type TEXT, - target_event_id TEXT, - sender TEXT, - room_id TEXT -); - -CREATE TABLE IF NOT EXISTS topics( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - topic TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); -CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); - -CREATE TABLE IF NOT EXISTS room_names( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - name TEXT NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); -CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); - -CREATE TABLE IF NOT EXISTS rooms( - room_id TEXT PRIMARY KEY NOT NULL, - is_public INTEGER, - creator TEXT -); - -CREATE TABLE IF NOT EXISTS room_hosts( - room_id TEXT NOT NULL, - host TEXT NOT NULL, - CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE -); - -CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/current/11/keys.sql b/synapse/storage/schema/current/11/keys.sql deleted file mode 100644 index a9e0a4fe0d..0000000000 --- a/synapse/storage/schema/current/11/keys.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name TEXT, -- Server name. - fingerprint TEXT, -- Certificate fingerprint. - from_server TEXT, -- Which key server the certificate was fetched from. - ts_added_ms INTEGER, -- When the certifcate was added. - tls_certificate BLOB, -- DER encoded x509 certificate. - CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) -); - -CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name TEXT, -- Server name. - key_id TEXT, -- Key version. - from_server TEXT, -- Which key server the key was fetched form. - ts_added_ms INTEGER, -- When the key was added. - verify_key BLOB, -- NACL verification key. - CONSTRAINT uniqueness UNIQUE (server_name, key_id) -); diff --git a/synapse/storage/schema/current/11/media_repository.sql b/synapse/storage/schema/current/11/media_repository.sql deleted file mode 100644 index afdf48cbfb..0000000000 --- a/synapse/storage/schema/current/11/media_repository.sql +++ /dev/null @@ -1,68 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id TEXT, -- The id used to refer to the media. - media_type TEXT, -- The MIME-type of the media. - media_length INTEGER, -- Length of the media in bytes. - created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. - user_id TEXT, -- The user who uploaded the file. - CONSTRAINT uniqueness UNIQUE (media_id) -); - -CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id TEXT, -- The id used to refer to the media. - thumbnail_width INTEGER, -- The width of the thumbnail in pixels. - thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type TEXT, -- The MIME-type of the thumbnail. - thumbnail_method TEXT, -- The method used to make the thumbnail. - thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - CONSTRAINT uniqueness UNIQUE ( - media_id, thumbnail_width, thumbnail_height, thumbnail_type - ) -); - -CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id - ON local_media_repository_thumbnails (media_id); - -CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media on that server. - media_type TEXT, -- The MIME-type of the media. - created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. - media_length INTEGER, -- Length of the media in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE (media_origin, media_id) -); - -CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media. - thumbnail_width INTEGER, -- The width of the thumbnail in pixels. - thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method TEXT, -- The method used to make the thumbnail - thumbnail_type TEXT, -- The MIME-type of the thumbnail. - thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE ( - media_origin, media_id, thumbnail_width, thumbnail_height, - thumbnail_type, thumbnail_type - ) -); - -CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id - ON local_media_repository_thumbnails (media_id); diff --git a/synapse/storage/schema/current/11/presence.sql b/synapse/storage/schema/current/11/presence.sql deleted file mode 100644 index f9f8db9697..0000000000 --- a/synapse/storage/schema/current/11/presence.sql +++ /dev/null @@ -1,38 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS presence( - user_id INTEGER NOT NULL, - state INTEGER, - status_msg TEXT, - mtime INTEGER, -- miliseconds since last state change - FOREIGN KEY(user_id) REFERENCES users(id) -); - --- For each of /my/ users which possibly-remote users are allowed to see their --- presence state -CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id INTEGER NOT NULL, - observer_user_id TEXT, -- a UserID, - FOREIGN KEY(observed_user_id) REFERENCES users(id) -); - --- For each of /my/ users (watcher), which possibly-remote users are they --- watching? -CREATE TABLE IF NOT EXISTS presence_list( - user_id INTEGER NOT NULL, - observed_user_id TEXT, -- a UserID, - accepted BOOLEAN, - FOREIGN KEY(user_id) REFERENCES users(id) -); diff --git a/synapse/storage/schema/current/11/profiles.sql b/synapse/storage/schema/current/11/profiles.sql deleted file mode 100644 index f06a528b4d..0000000000 --- a/synapse/storage/schema/current/11/profiles.sql +++ /dev/null @@ -1,20 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS profiles( - user_id INTEGER NOT NULL, - displayname TEXT, - avatar_url TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) -); diff --git a/synapse/storage/schema/current/11/redactions.sql b/synapse/storage/schema/current/11/redactions.sql deleted file mode 100644 index 5011d95db8..0000000000 --- a/synapse/storage/schema/current/11/redactions.sql +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS redactions ( - event_id TEXT NOT NULL, - redacts TEXT NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); -CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/current/11/room_aliases.sql b/synapse/storage/schema/current/11/room_aliases.sql deleted file mode 100644 index 0d2df01603..0000000000 --- a/synapse/storage/schema/current/11/room_aliases.sql +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS room_aliases( - room_alias TEXT NOT NULL, - room_id TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias TEXT NOT NULL, - server TEXT NOT NULL -); - - - diff --git a/synapse/storage/schema/current/11/state.sql b/synapse/storage/schema/current/11/state.sql deleted file mode 100644 index 1fe8f1e430..0000000000 --- a/synapse/storage/schema/current/11/state.sql +++ /dev/null @@ -1,47 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS state_groups( - id INTEGER PRIMARY KEY, - room_id TEXT NOT NULL, - event_id TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS state_groups_state( - state_group INTEGER NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - event_id TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id TEXT NOT NULL, - state_group INTEGER NOT NULL, - CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); - -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state( - state_group -); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state( - room_id, type, state_key -); - -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups( - event_id -); \ No newline at end of file diff --git a/synapse/storage/schema/current/11/transactions.sql b/synapse/storage/schema/current/11/transactions.sql deleted file mode 100644 index 2d30f99b06..0000000000 --- a/synapse/storage/schema/current/11/transactions.sql +++ /dev/null @@ -1,68 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- Stores what transaction ids we have received and what our response was -CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id TEXT, - origin TEXT, - ts INTEGER, - response_code INTEGER, - response_json TEXT, - has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx - CONSTRAINT uniquesss UNIQUE (transaction_id, origin) ON CONFLICT REPLACE -); - -CREATE UNIQUE INDEX IF NOT EXISTS transactions_txid ON received_transactions(transaction_id, origin); -CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; - - --- Stores what transactions we've sent, what their response was (if we got one) and whether we have --- since referenced the transaction in another outgoing transaction -CREATE TABLE IF NOT EXISTS sent_transactions( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering - transaction_id TEXT, - destination TEXT, - response_code INTEGER DEFAULT 0, - response_json TEXT, - ts INTEGER -); - -CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( - destination -); -CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); --- So that we can do an efficient look up of all transactions that have yet to be successfully --- sent. -CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); - - --- For sent transactions only. -CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( - transaction_id INTEGER, - destination TEXT, - pdu_id TEXT, - pdu_origin TEXT -); - -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination); - --- To track destination health -CREATE TABLE IF NOT EXISTS destinations( - destination TEXT PRIMARY KEY, - retry_last_ts INTEGER, - retry_interval INTEGER -); diff --git a/synapse/storage/schema/current/11/users.sql b/synapse/storage/schema/current/11/users.sql deleted file mode 100644 index 08ccfdac0a..0000000000 --- a/synapse/storage/schema/current/11/users.sql +++ /dev/null @@ -1,45 +0,0 @@ -/* Copyright 2014, 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS users( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT, - password_hash TEXT, - creation_ts INTEGER, - admin BOOL DEFAULT 0 NOT NULL, - UNIQUE(name) ON CONFLICT ROLLBACK -); - -CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id INTEGER NOT NULL, - device_id TEXT, - token TEXT NOT NULL, - last_used INTEGER, - FOREIGN KEY(user_id) REFERENCES users(id), - UNIQUE(token) ON CONFLICT ROLLBACK -); - -CREATE TABLE IF NOT EXISTS user_ips ( - user TEXT NOT NULL, - access_token TEXT NOT NULL, - device_id TEXT, - ip TEXT NOT NULL, - user_agent TEXT NOT NULL, - last_seen INTEGER NOT NULL, - CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE -); - -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); - -- cgit 1.4.1 From 8c8354e85a702ef36de15db50d294e1a724ced7d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 13:34:38 +0000 Subject: Actually add full_schemas dir --- .../storage/schema/full_schemas/11/event_edges.sql | 89 +++++++++++++++ .../schema/full_schemas/11/event_signatures.sql | 65 +++++++++++ synapse/storage/schema/full_schemas/11/im.sql | 125 +++++++++++++++++++++ synapse/storage/schema/full_schemas/11/keys.sql | 31 +++++ .../schema/full_schemas/11/media_repository.sql | 68 +++++++++++ .../storage/schema/full_schemas/11/presence.sql | 38 +++++++ .../storage/schema/full_schemas/11/profiles.sql | 20 ++++ .../storage/schema/full_schemas/11/redactions.sql | 22 ++++ .../schema/full_schemas/11/room_aliases.sql | 27 +++++ synapse/storage/schema/full_schemas/11/state.sql | 47 ++++++++ .../schema/full_schemas/11/transactions.sql | 68 +++++++++++ synapse/storage/schema/full_schemas/11/users.sql | 45 ++++++++ 12 files changed, 645 insertions(+) create mode 100644 synapse/storage/schema/full_schemas/11/event_edges.sql create mode 100644 synapse/storage/schema/full_schemas/11/event_signatures.sql create mode 100644 synapse/storage/schema/full_schemas/11/im.sql create mode 100644 synapse/storage/schema/full_schemas/11/keys.sql create mode 100644 synapse/storage/schema/full_schemas/11/media_repository.sql create mode 100644 synapse/storage/schema/full_schemas/11/presence.sql create mode 100644 synapse/storage/schema/full_schemas/11/profiles.sql create mode 100644 synapse/storage/schema/full_schemas/11/redactions.sql create mode 100644 synapse/storage/schema/full_schemas/11/room_aliases.sql create mode 100644 synapse/storage/schema/full_schemas/11/state.sql create mode 100644 synapse/storage/schema/full_schemas/11/transactions.sql create mode 100644 synapse/storage/schema/full_schemas/11/users.sql (limited to 'synapse') diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql new file mode 100644 index 0000000000..1e766d6db2 --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -0,0 +1,89 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_forward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_backward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_edges( + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + is_state INTEGER NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state) +); + +CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); +CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); + + +CREATE TABLE IF NOT EXISTS room_depth( + room_id TEXT NOT NULL, + min_depth INTEGER NOT NULL, + CONSTRAINT uniqueness UNIQUE (room_id) +); + +CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); + + +create TABLE IF NOT EXISTS event_destinations( + event_id TEXT NOT NULL, + destination TEXT NOT NULL, + delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered + CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); + + +CREATE TABLE IF NOT EXISTS state_forward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( + room_id, type, state_key +); +CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_auth( + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id) +); + +CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); +CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql new file mode 100644 index 0000000000..c28c39c48a --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -0,0 +1,65 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_content_hashes ( + event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_reference_hashes ( + event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes ( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_signatures ( + event_id TEXT, + signature_name TEXT, + key_id TEXT, + signature BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id) +); + +CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_edge_hashes( + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE ( + event_id, prev_event_id, algorithm + ) +); + +CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes( + event_id +); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql new file mode 100644 index 0000000000..dd00c1cd2f --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -0,0 +1,125 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS events( + stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, + topological_ordering INTEGER NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, + content TEXT NOT NULL, + unrecognized_keys TEXT, + processed BOOL NOT NULL, + outlier BOOL NOT NULL, + depth INTEGER DEFAULT 0 NOT NULL, + CONSTRAINT ev_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); +CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); +CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); +CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); + + +CREATE TABLE IF NOT EXISTS event_json( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + internal_metadata NOT NULL, + json BLOB NOT NULL, + CONSTRAINT ev_j_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); +CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); + + +CREATE TABLE IF NOT EXISTS state_events( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT +); + +CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id); +CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); +CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); +CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); + + +CREATE TABLE IF NOT EXISTS current_state_events( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); +CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); +CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); + +CREATE TABLE IF NOT EXISTS room_memberships( + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); +CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); + +CREATE TABLE IF NOT EXISTS feedback( + event_id TEXT NOT NULL, + feedback_type TEXT, + target_event_id TEXT, + sender TEXT, + room_id TEXT +); + +CREATE TABLE IF NOT EXISTS topics( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + topic TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); +CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); + +CREATE TABLE IF NOT EXISTS room_names( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + name TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); +CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); + +CREATE TABLE IF NOT EXISTS rooms( + room_id TEXT PRIMARY KEY NOT NULL, + is_public INTEGER, + creator TEXT +); + +CREATE TABLE IF NOT EXISTS room_hosts( + room_id TEXT NOT NULL, + host TEXT NOT NULL, + CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE +); + +CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql new file mode 100644 index 0000000000..a9e0a4fe0d --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -0,0 +1,31 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS server_tls_certificates( + server_name TEXT, -- Server name. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. + ts_added_ms INTEGER, -- When the certifcate was added. + tls_certificate BLOB, -- DER encoded x509 certificate. + CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) +); + +CREATE TABLE IF NOT EXISTS server_signature_keys( + server_name TEXT, -- Server name. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. + ts_added_ms INTEGER, -- When the key was added. + verify_key BLOB, -- NACL verification key. + CONSTRAINT uniqueness UNIQUE (server_name, key_id) +); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql new file mode 100644 index 0000000000..afdf48cbfb --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -0,0 +1,68 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS local_media_repository ( + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. + media_length INTEGER, -- Length of the media in bytes. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. + CONSTRAINT uniqueness UNIQUE (media_id) +); + +CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + CONSTRAINT uniqueness UNIQUE ( + media_id, thumbnail_width, thumbnail_height, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + +CREATE TABLE IF NOT EXISTS remote_media_cache ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + media_length INTEGER, -- Length of the media in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE (media_origin, media_id) +); + +CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE ( + media_origin, media_id, thumbnail_width, thumbnail_height, + thumbnail_type, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql new file mode 100644 index 0000000000..f9f8db9697 --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -0,0 +1,38 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS presence( + user_id INTEGER NOT NULL, + state INTEGER, + status_msg TEXT, + mtime INTEGER, -- miliseconds since last state change + FOREIGN KEY(user_id) REFERENCES users(id) +); + +-- For each of /my/ users which possibly-remote users are allowed to see their +-- presence state +CREATE TABLE IF NOT EXISTS presence_allow_inbound( + observed_user_id INTEGER NOT NULL, + observer_user_id TEXT, -- a UserID, + FOREIGN KEY(observed_user_id) REFERENCES users(id) +); + +-- For each of /my/ users (watcher), which possibly-remote users are they +-- watching? +CREATE TABLE IF NOT EXISTS presence_list( + user_id INTEGER NOT NULL, + observed_user_id TEXT, -- a UserID, + accepted BOOLEAN, + FOREIGN KEY(user_id) REFERENCES users(id) +); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql new file mode 100644 index 0000000000..f06a528b4d --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -0,0 +1,20 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS profiles( + user_id INTEGER NOT NULL, + displayname TEXT, + avatar_url TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql new file mode 100644 index 0000000000..5011d95db8 --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -0,0 +1,22 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS redactions ( + event_id TEXT NOT NULL, + redacts TEXT NOT NULL, + CONSTRAINT ev_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); +CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql new file mode 100644 index 0000000000..0d2df01603 --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -0,0 +1,27 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS room_aliases( + room_alias TEXT NOT NULL, + room_id TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS room_alias_servers( + room_alias TEXT NOT NULL, + server TEXT NOT NULL +); + + + diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql new file mode 100644 index 0000000000..1fe8f1e430 --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -0,0 +1,47 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS state_groups( + id INTEGER PRIMARY KEY, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS state_groups_state( + state_group INTEGER NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS event_to_state_groups( + event_id TEXT NOT NULL, + state_group INTEGER NOT NULL, + CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); + +CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state( + state_group +); +CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state( + room_id, type, state_key +); + +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups( + event_id +); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql new file mode 100644 index 0000000000..2d30f99b06 --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -0,0 +1,68 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Stores what transaction ids we have received and what our response was +CREATE TABLE IF NOT EXISTS received_transactions( + transaction_id TEXT, + origin TEXT, + ts INTEGER, + response_code INTEGER, + response_json TEXT, + has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + CONSTRAINT uniquesss UNIQUE (transaction_id, origin) ON CONFLICT REPLACE +); + +CREATE UNIQUE INDEX IF NOT EXISTS transactions_txid ON received_transactions(transaction_id, origin); +CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; + + +-- Stores what transactions we've sent, what their response was (if we got one) and whether we have +-- since referenced the transaction in another outgoing transaction +CREATE TABLE IF NOT EXISTS sent_transactions( + id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering + transaction_id TEXT, + destination TEXT, + response_code INTEGER DEFAULT 0, + response_json TEXT, + ts INTEGER +); + +CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( + destination +); +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +-- So that we can do an efficient look up of all transactions that have yet to be successfully +-- sent. +CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); + + +-- For sent transactions only. +CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( + transaction_id INTEGER, + destination TEXT, + pdu_id TEXT, + pdu_origin TEXT +); + +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination); + +-- To track destination health +CREATE TABLE IF NOT EXISTS destinations( + destination TEXT PRIMARY KEY, + retry_last_ts INTEGER, + retry_interval INTEGER +); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql new file mode 100644 index 0000000000..08ccfdac0a --- /dev/null +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -0,0 +1,45 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS users( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + password_hash TEXT, + creation_ts INTEGER, + admin BOOL DEFAULT 0 NOT NULL, + UNIQUE(name) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS access_tokens( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + device_id TEXT, + token TEXT NOT NULL, + last_used INTEGER, + FOREIGN KEY(user_id) REFERENCES users(id), + UNIQUE(token) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS user_ips ( + user TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, + user_agent TEXT NOT NULL, + last_seen INTEGER NOT NULL, + CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); + -- cgit 1.4.1 From 640e53935dc2585c7a6365980cd288e5ed176b38 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 13:43:17 +0000 Subject: Use context manager with db conn to correctly commit and rollback --- synapse/storage/__init__.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a22a0e2f39..d637e8c4f7 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -55,7 +55,7 @@ import re logger = logging.getLogger(__name__) -# Remember to update this number every time an change is made to database +# Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. SCHEMA_VERSION = 14 @@ -583,19 +583,19 @@ def prepare_database(db_conn): """Prepares a database for usage. Will either create all necessary tables or upgrade from an older schema version. """ - cur = db_conn.cursor() - version_info = _get_or_create_schema_state(cur) + with db_conn: + cur = db_conn.cursor() + version_info = _get_or_create_schema_state(cur) - if version_info: - user_version, delta_files, upgraded = version_info - _upgrade_existing_database(cur, user_version, delta_files, upgraded) - else: - _setup_new_database(cur) + if version_info: + user_version, delta_files, upgraded = version_info + _upgrade_existing_database(cur, user_version, delta_files, upgraded) + else: + _setup_new_database(cur) - cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) - db_conn.commit() + cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) - cur.close() + cur.close() def _setup_new_database(cur): -- cgit 1.4.1 From b4c38738f438ee2b035a92d6e26b0f078956575f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 13:43:35 +0000 Subject: Change to use logger in db upgrade script --- synapse/storage/schema/delta/14/upgrade_appservice_db.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/schema/delta/14/upgrade_appservice_db.py b/synapse/storage/schema/delta/14/upgrade_appservice_db.py index 55e43c41ab..847b1c5b89 100644 --- a/synapse/storage/schema/delta/14/upgrade_appservice_db.py +++ b/synapse/storage/schema/delta/14/upgrade_appservice_db.py @@ -1,11 +1,14 @@ import json +import logging + +logger = logging.getLogger(__name__) def run_upgrade(cur): cur.execute("SELECT id, regex FROM application_services_regex") for row in cur.fetchall(): try: - print "checking %s..." % row[0] + logger.debug("Checking %s..." % row[0]) json.loads(row[1]) except ValueError: # row isn't in json, make it so. -- cgit 1.4.1 From 5eefd1f618a19cc27cfdaf915dc3abd3720cd0e7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 13:52:18 +0000 Subject: Add unique constraint on schema_version.lock schema. Use conflict clause in sql. --- synapse/storage/__init__.py | 6 +++--- synapse/storage/schema/schema_version.sql | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d637e8c4f7..c09228c37f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -645,7 +645,7 @@ def _setup_new_database(cur): cur.executescript(sql_script) cur.execute( - "INSERT INTO schema_version (version, upgraded)" + "INSERT OR REPLACE INTO schema_version (version, upgraded)" " VALUES (?,?)", (max_current_ver, False) ) @@ -748,7 +748,7 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded): ) cur.execute( - "INSERT INTO schema_version (version, upgraded)" + "INSERT OR REPLACE INTO schema_version (version, upgraded)" " VALUES (?,?)", (v, True) ) @@ -803,7 +803,7 @@ def prepare_sqlite3_database(db_conn): if row and row[0]: db_conn.execute( - "INSERT INTO schema_version (version, upgraded)" + "INSERT OR REPLACE INTO schema_version (version, upgraded)" " VALUES (?,?)", (row[0], False) ) diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index 83a8c7b7ce..20b1481ba5 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -17,7 +17,8 @@ CREATE TABLE IF NOT EXISTS schema_version( Lock char(1) NOT NULL DEFAULT 'X', -- Makes sure this table only has one row. version INTEGER NOT NULL, upgraded BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. - CONSTRAINT schema_version_lock CHECK (Lock='X') ON CONFLICT REPLACE + CONSTRAINT schema_version_lock_x CHECK (Lock='X') + CONSTRAINT schema_version_lock_uniq UNIQUE (Lock) ); CREATE TABLE IF NOT EXISTS schema_deltas( -- cgit 1.4.1 From 16dd87d848f686835cbff82333e00c6411b1436b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 14:03:21 +0000 Subject: Don't assume db conn is a Context Manager. Twisted adbapi wrapped connections aren't context managers. --- synapse/storage/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c09228c37f..87189e54b8 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -583,7 +583,7 @@ def prepare_database(db_conn): """Prepares a database for usage. Will either create all necessary tables or upgrade from an older schema version. """ - with db_conn: + try: cur = db_conn.cursor() version_info = _get_or_create_schema_state(cur) @@ -596,6 +596,10 @@ def prepare_database(db_conn): cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) cur.close() + db_conn.commit() + except: + db_conn.rollback() + raise def _setup_new_database(cur): -- cgit 1.4.1 From 2a45f3d448439ebef047cc09ac62bccfa3ebac5a Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 Mar 2015 14:17:59 +0000 Subject: Use if not results rather than len, as per feedback. --- synapse/storage/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index ea865b6abf..bbf322cc84 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -63,7 +63,7 @@ class PushRuleStore(SQLBaseStore): {'user_name': user_name, 'rule_id': rule_id}, ['enabled'] ) - if len(results) == 0: + if not results: defer.returnValue(True) defer.returnValue(results[0]) -- cgit 1.4.1 From f701197227998e9fe270034be3c053cfa1f12ccf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 14:20:14 +0000 Subject: Add example directory structures in doc --- synapse/storage/__init__.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 87189e54b8..8b2f8f8963 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -608,10 +608,27 @@ def _setup_new_database(cur): The "full_schemas" directory has subdirectories named after versions. This function searches for the highest version less than or equal to - `SCHEMA_VERSION` and excutes all .sql files in that directory. + `SCHEMA_VERSION` and executes all .sql files in that directory. The function will then apply all deltas for all versions after the base version. + + Example directory structure: + + schema/ + delta/ + ... + full_schemas/ + 3/ + test.sql + ... + 11/ + foo.sql + bar.sql + ... + + In the example foo.sql and bar.sql would be run, and then any delta files + for versions strictly greater than 11. """ current_dir = os.path.join(dir_path, "schema", "full_schemas") directory_entries = os.listdir(current_dir) @@ -675,6 +692,24 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded): This is a no-op of current_version == SCHEMA_VERSION. + Example directory structure: + + schema/ + delta/ + 11/ + foo.sql + ... + 12/ + foo.sql + bar.py + ... + full_schemas/ + ... + + In the example, if current_version is 11, then foo.sql will be run if and + only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in + some arbitrary order. + Args: cur (Cursor) current_version (int): The current version of the schema -- cgit 1.4.1 From 5681264faa1457ac67a66ad2474ef644f0efeb55 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 14:21:53 +0000 Subject: s/%r/%s/ --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 8b2f8f8963..98b877bd08 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -659,7 +659,7 @@ def _setup_new_database(cur): sql_script = "BEGIN TRANSACTION;\n" for filename in fnmatch.filter(directory_entries, "*.sql"): sql_loc = os.path.join(sql_dir, filename) - logger.debug("Applying schema %r", sql_loc) + logger.debug("Applying schema %s", sql_loc) sql_script += read_schema(sql_loc) sql_script += "\n" sql_script += "COMMIT TRANSACTION;" -- cgit 1.4.1 From 17d319a20df6e92ddbb8ca5b9e08615d2975b466 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 15:06:22 +0000 Subject: s/schema_deltas/applied_schema_deltas/ --- synapse/storage/__init__.py | 16 +++++++++------- synapse/storage/schema/schema_version.sql | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 98b877bd08..e2d5e5a41d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -674,12 +674,13 @@ def _setup_new_database(cur): _upgrade_existing_database( cur, current_version=max_current_ver, - delta_files=[], + applied_delta_files=[], upgraded=False ) -def _upgrade_existing_database(cur, current_version, delta_files, upgraded): +def _upgrade_existing_database(cur, current_version, applied_delta_files, + upgraded): """Upgrades an existing database. Delta files can either be SQL stored in *.sql files, or python modules @@ -712,8 +713,9 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded): Args: cur (Cursor) - current_version (int): The current version of the schema - delta_files (list): A list of deltas that have already been applied + current_version (int): The current version of the schema. + applied_delta_files (list): A list of deltas that have already been + applied. upgraded (bool): Whether the current version was generated by having applied deltas or from full schema file. If `True` the function will never apply delta files for the given `current_version`, since @@ -746,7 +748,7 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded): directory_entries.sort() for file_name in directory_entries: relative_path = os.path.join(str(v), file_name) - if relative_path in delta_files: + if relative_path in applied_delta_files: continue absolute_path = os.path.join( @@ -781,7 +783,7 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded): # Mark as done. cur.execute( - "INSERT INTO schema_deltas (version, file)" + "INSERT INTO applied_schema_deltas (version, file)" " VALUES (?,?)", (v, relative_path) ) @@ -807,7 +809,7 @@ def _get_or_create_schema_state(txn): if current_version: txn.execute( - "SELECT file FROM schema_deltas WHERE version >= ?", + "SELECT file FROM applied_schema_deltas WHERE version >= ?", (current_version,) ) return current_version, txn.fetchall(), upgraded diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index 20b1481ba5..0431e2d051 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -21,10 +21,10 @@ CREATE TABLE IF NOT EXISTS schema_version( CONSTRAINT schema_version_lock_uniq UNIQUE (Lock) ); -CREATE TABLE IF NOT EXISTS schema_deltas( +CREATE TABLE IF NOT EXISTS applied_schema_deltas( version INTEGER NOT NULL, file TEXT NOT NULL, CONSTRAINT schema_deltas_ver_file UNIQUE (version, file) ON CONFLICT IGNORE ); -CREATE INDEX IF NOT EXISTS schema_deltas_ver ON schema_deltas(version); +CREATE INDEX IF NOT EXISTS schema_deltas_ver ON applied_schema_deltas(version); -- cgit 1.4.1 From d56c01fff46d9adbccb8149d61f318f11040775e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Mar 2015 15:10:05 +0000 Subject: Note that we don't specify execution order --- synapse/storage/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e2d5e5a41d..a3ff995695 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -691,6 +691,10 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, even if there has been no version bump. This is useful for development where orthogonal schema changes may happen on separate branches. + Different delta files for the same version *must* be orthogonal and give + the same result when applied in any order. No guarantees are made on the + order of execution of these scripts. + This is a no-op of current_version == SCHEMA_VERSION. Example directory structure: -- cgit 1.4.1 From 590b544f678cba7a364f2ddf205993d50feba76e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 Mar 2015 15:29:02 +0000 Subject: Add default rule to suppress notices. --- synapse/push/baserules.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index c8c19f2a18..91c528cc19 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -104,6 +104,19 @@ def make_base_override_rules(): def make_base_underride_rules(): return [ + { + 'rule_id': 'global/underride/.m.rule.suppress_notices', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'content.msgtype', + 'pattern': 'm.notice', + } + ], + 'actions': [ + 'dont-notify', + ] + }, { 'rule_id': 'global/underride/.m.rule.fallback', 'conditions': [ -- cgit 1.4.1 From 027fd1242cfaf3ae26856f94c01d2b7c04705172 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 4 Mar 2015 17:32:28 +0000 Subject: Give LruCache a __len__, so that len(cache) works --- synapse/util/lrucache.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'synapse') diff --git a/synapse/util/lrucache.py b/synapse/util/lrucache.py index a45c673d32..f115f50e50 100644 --- a/synapse/util/lrucache.py +++ b/synapse/util/lrucache.py @@ -88,11 +88,15 @@ class LruCache(object): else: return default + def cache_len(): + return len(cache) + self.sentinel = object() self.get = cache_get self.set = cache_set self.setdefault = cache_set_default self.pop = cache_pop + self.len = cache_len def __getitem__(self, key): result = self.get(key, self.sentinel) @@ -108,3 +112,6 @@ class LruCache(object): result = self.pop(key, self.sentinel) if result is self.sentinel: raise KeyError() + + def __len__(self): + return self.len() -- cgit 1.4.1 From dc4b774f1e984711eccde0faa4bbf9463ecad964 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 5 Mar 2015 14:30:20 +0000 Subject: Rename rooms_to_listeners to room_to_listeners, for consistency with user_ and appservice_* --- synapse/notifier.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/notifier.py b/synapse/notifier.py index 09d23e79b8..df13e8ddb6 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -63,7 +63,7 @@ class _NotificationListener(object): pass for room in self.rooms: - lst = notifier.rooms_to_listeners.get(room, set()) + lst = notifier.room_to_listeners.get(room, set()) lst.discard(self) notifier.user_to_listeners.get(self.user, set()).discard(self) @@ -83,7 +83,7 @@ class Notifier(object): def __init__(self, hs): self.hs = hs - self.rooms_to_listeners = {} + self.room_to_listeners = {} self.user_to_listeners = {} self.appservice_to_listeners = {} @@ -116,17 +116,17 @@ class Notifier(object): room_source = self.event_sources.sources["room"] - listeners = self.rooms_to_listeners.get(room_id, set()).copy() + listeners = self.room_to_listeners.get(room_id, set()).copy() for user in extra_users: listeners |= self.user_to_listeners.get(user, set()).copy() for appservice in self.appservice_to_listeners: # TODO (kegan): Redundant appservice listener checks? - # App services will already be in the rooms_to_listeners set, but + # App services will already be in the room_to_listeners set, but # that isn't enough. They need to be checked here in order to # receive *invites* for users they are interested in. Does this - # make the rooms_to_listeners check somewhat obselete? + # make the room_to_listeners check somewhat obselete? if appservice.is_interested(event): listeners |= self.appservice_to_listeners.get( appservice, set() @@ -184,7 +184,7 @@ class Notifier(object): listeners |= self.user_to_listeners.get(user, set()).copy() for room in rooms: - listeners |= self.rooms_to_listeners.get(room, set()).copy() + listeners |= self.room_to_listeners.get(room, set()).copy() @defer.inlineCallbacks def notify(listener): @@ -337,7 +337,7 @@ class Notifier(object): @log_function def _register_with_keys(self, listener): for room in listener.rooms: - s = self.rooms_to_listeners.setdefault(room, set()) + s = self.room_to_listeners.setdefault(room, set()) s.add(listener) self.user_to_listeners.setdefault(listener.user, set()).add(listener) @@ -380,5 +380,5 @@ class Notifier(object): def _user_joined_room(self, user, room_id): new_listeners = self.user_to_listeners.get(user, set()) - listeners = self.rooms_to_listeners.setdefault(room_id, set()) + listeners = self.room_to_listeners.setdefault(room_id, set()) listeners |= new_listeners -- cgit 1.4.1 From ae702d161ab6d518caa91759ec6bdec01b11954f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Mar 2015 16:08:02 +0000 Subject: Handle if get_missing_pdu returns 400 or not all events. --- synapse/federation/federation_client.py | 109 ++++++++++++++++++++++++++++---- synapse/federation/federation_server.py | 6 +- 2 files changed, 100 insertions(+), 15 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index ca89a0787c..b87c8a3bbb 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -19,14 +19,18 @@ from twisted.internet import defer from .federation_base import FederationBase from .units import Edu -from synapse.api.errors import CodeMessageException, SynapseError +from synapse.api.errors import ( + CodeMessageException, HttpResponseException, SynapseError, +) from synapse.util.expiringcache import ExpiringCache from synapse.util.logutils import log_function from synapse.events import FrozenEvent from synapse.util.retryutils import get_retry_limiter, NotRetryingDestination +import itertools import logging +import random logger = logging.getLogger(__name__) @@ -440,21 +444,100 @@ class FederationClient(FederationBase): defer.returnValue(ret) @defer.inlineCallbacks - def get_missing_events(self, destination, room_id, earliest_events, + def get_missing_events(self, destination, room_id, earliest_events_ids, latest_events, limit, min_depth): - content = yield self.transport_layer.get_missing_events( - destination, room_id, earliest_events, latest_events, limit, - min_depth, - ) + try: + content = yield self.transport_layer.get_missing_events( + destination=destination, + room_id=room_id, + earliest_events=earliest_events_ids, + latest_events=[e.event_id for e in latest_events], + limit=limit, + min_depth=min_depth, + ) + + events = [ + self.event_from_pdu_json(e) + for e in content.get("events", []) + ] + + signed_events = yield self._check_sigs_and_hash_and_fetch( + destination, events, outlier=True + ) + + have_gotten_all_from_destination = True + except HttpResponseException as e: + if not e.code == 400: + raise - events = [ - self.event_from_pdu_json(e) - for e in content.get("events", []) - ] + signed_events = [] + have_gotten_all_from_destination = False - signed_events = yield self._check_sigs_and_hash_and_fetch( - destination, events, outlier=True - ) + if len(signed_events) >= limit: + defer.returnValue(signed_events) + + servers = yield self.store.get_joined_hosts_for_room(room_id) + + servers = set(servers) + servers.discard(self.server_name) + + failed_to_fetch = set() + + while len(signed_events) < limit: + # Are we missing any? + + seen_events = set(earliest_events_ids) + seen_events.update(e.event_id for e in signed_events) + + missing_events = {} + for e in itertools.chain(latest_events, signed_events): + missing_events.update({ + e_id: e.depth for e_id, _ in e.prev_events + if e_id not in seen_events and e_id not in failed_to_fetch + }) + + if not missing_events: + break + + have_seen = yield self.store.have_events(missing_events) + + for k in have_seen: + missing_events.pop(k, None) + + if not missing_events: + break + + # Okay, we haven't gotten everything yet. Lets get them. + ordered_missing = sorted(missing_events.items(), key=lambda x: x[0]) + + if have_gotten_all_from_destination: + servers.discard(destination) + + def random_server_list(): + srvs = list(servers) + random.shuffle(srvs) + return srvs + + deferreds = [ + self.get_pdu( + destinations=random_server_list(), + event_id=e_id, + ) + for e_id, depth in ordered_missing[:limit - len(signed_events)] + ] + + got_a_new_event = False + + res = yield defer.DeferredList(deferreds, consumeErrors=True) + for (result, val), (e_id, _) in zip(res, ordered_missing): + if result: + signed_events.append(val) + got_a_new_event = True + else: + failed_to_fetch.add(e_id) + + if not got_a_new_event: + break defer.returnValue(signed_events) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 4264d857be..dd4ca74ba6 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -413,12 +413,14 @@ class FederationServer(FederationBase): missing_events = yield self.get_missing_events( origin, pdu.room_id, - earliest_events=list(latest), - latest_events=[pdu.event_id], + earliest_events_ids=list(latest), + latest_events=[pdu], limit=10, min_depth=min_depth, ) + missing_events.sort(key=lambda x: x.depth) + for e in missing_events: yield self._handle_new_pdu( origin, -- cgit 1.4.1 From 9d9d39536b8be554eec2b16b2846ad31ac643721 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 5 Mar 2015 16:24:13 +0000 Subject: Slightly reduce the insane amounts of indentation in main http server response path, by 'continue'ing around a non-match or falling through --- synapse/http/server.py | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) (limited to 'synapse') diff --git a/synapse/http/server.py b/synapse/http/server.py index 74a101a5d7..767c3ef79b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -124,27 +124,29 @@ class JsonResource(HttpServer, resource.Resource): # and path regex match for path_entry in self.path_regexs.get(request.method, []): m = path_entry.pattern.match(request.path) - if m: - # We found a match! Trigger callback and then return the - # returned response. We pass both the request and any - # matched groups from the regex to the callback. - - args = [ - urllib.unquote(u).decode("UTF-8") for u in m.groups() - ] - - logger.info( - "Received request: %s %s", - request.method, request.path - ) - - code, response = yield path_entry.callback( - request, - *args - ) - - self._send_response(request, code, response) - return + if not m: + continue + + # We found a match! Trigger callback and then return the + # returned response. We pass both the request and any + # matched groups from the regex to the callback. + + args = [ + urllib.unquote(u).decode("UTF-8") for u in m.groups() + ] + + logger.info( + "Received request: %s %s", + request.method, request.path + ) + + code, response = yield path_entry.callback( + request, + *args + ) + + self._send_response(request, code, response) + return # Huh. No one wanted to handle that? Fiiiiiine. Send 400. raise UnrecognizedRequestError() -- cgit 1.4.1 From 6dfd8c73fcdd727cd6589513e2b8059f779623ae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Mar 2015 16:31:13 +0000 Subject: Docs. --- synapse/federation/federation_client.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index b87c8a3bbb..11e2753fed 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -446,6 +446,20 @@ class FederationClient(FederationBase): @defer.inlineCallbacks def get_missing_events(self, destination, room_id, earliest_events_ids, latest_events, limit, min_depth): + """Tries to fetch events we are missing. This is called when we receive + an event without having received all of its ancestors. + + Args: + destination (str) + room_id (str) + earliest_events_ids (list): List of event ids. Effectively the + events we expected to receive, but haven't. `get_missing_events` + should only return events that didn't happen before these. + latest_events (list): List of events we have received that we don't + have all previous events for. + limit (int): Maximum number of events to return. + min_depth (int): Minimum depth of events tor return. + """ try: content = yield self.transport_layer.get_missing_events( destination=destination, @@ -470,6 +484,8 @@ class FederationClient(FederationBase): if not e.code == 400: raise + # We are probably hitting an old server that doesn't support + # get_missing_events signed_events = [] have_gotten_all_from_destination = False -- cgit 1.4.1 From 39aa968a764816632a05ac0e3cf9c865b7a3a68d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Mar 2015 16:31:32 +0000 Subject: Respect min_depth argument --- synapse/federation/federation_client.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 11e2753fed..75b6a7b46a 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -507,10 +507,12 @@ class FederationClient(FederationBase): missing_events = {} for e in itertools.chain(latest_events, signed_events): - missing_events.update({ - e_id: e.depth for e_id, _ in e.prev_events - if e_id not in seen_events and e_id not in failed_to_fetch - }) + if e.depth > min_depth: + missing_events.update({ + e_id: e.depth for e_id, _ in e.prev_events + if e_id not in seen_events + and e_id not in failed_to_fetch + }) if not missing_events: break -- cgit 1.4.1 From 96fee64421a534787e9316a61ab407b43c782dc7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Mar 2015 16:31:47 +0000 Subject: Remove unecessary check --- synapse/federation/federation_client.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 75b6a7b46a..f131941f45 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -544,19 +544,13 @@ class FederationClient(FederationBase): for e_id, depth in ordered_missing[:limit - len(signed_events)] ] - got_a_new_event = False - res = yield defer.DeferredList(deferreds, consumeErrors=True) for (result, val), (e_id, _) in zip(res, ordered_missing): if result: signed_events.append(val) - got_a_new_event = True else: failed_to_fetch.add(e_id) - if not got_a_new_event: - break - defer.returnValue(signed_events) def event_from_pdu_json(self, pdu_json, outlier=False): -- cgit 1.4.1 From 9708f49abfb5fa48c1190364093ab4ce5c4e6f23 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Mar 2015 16:35:16 +0000 Subject: Docs --- synapse/federation/federation_server.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index dd4ca74ba6..9c7dcdba96 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -419,6 +419,8 @@ class FederationServer(FederationBase): min_depth=min_depth, ) + # We want to sort these by depth so we process them and + # tell clients about them in order. missing_events.sort(key=lambda x: x.depth) for e in missing_events: -- cgit 1.4.1 From 5b5c7a28d675f2c9a055153089aab96cac28c523 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Mar 2015 17:09:13 +0000 Subject: Log error message when we fail to fetch remote server keys --- synapse/crypto/keyring.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 828aced44a..f4db7b8a05 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -50,18 +50,27 @@ class Keyring(object): ) try: verify_key = yield self.get_server_verify_key(server_name, key_ids) - except IOError: + except IOError as e: + logger.warn( + "Got IOError when downloading keys for %s: %s %s", + server_name, type(e).__name__, str(e.message), + ) raise SynapseError( 502, "Error downloading keys for %s" % (server_name,), Codes.UNAUTHORIZED, ) - except: + except Exception as e: + logger.warn( + "Got Exception when downloading keys for %s: %s %s", + server_name, type(e).__name__, str(e.message), + ) raise SynapseError( 401, "No key for %s with id %s" % (server_name, key_ids), Codes.UNAUTHORIZED, ) + try: verify_signed_json(json_object, server_name, verify_key) except: -- cgit 1.4.1 From 130df8fb01bce1bf244940131d85d1c1a54ed5ef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Mar 2015 10:25:36 +0000 Subject: Add some randomness to the user specified timeout on event streams to mitigate against thundering herds problems --- synapse/handlers/events.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'synapse') diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 8d5f5c8499..d3297b7292 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -23,6 +23,7 @@ from synapse.events.utils import serialize_event from ._base import BaseHandler import logging +import random logger = logging.getLogger(__name__) @@ -72,6 +73,14 @@ class EventStreamHandler(BaseHandler): rm_handler = self.hs.get_handlers().room_member_handler room_ids = yield rm_handler.get_rooms_for_user(auth_user) + if timeout: + # If they've set a timeout set a minimum limit. + timeout = max(timeout, 500) + + # Add some randomness to this value to try and mitigate against + # thundering herds on restart. + timeout = random.randint(int(timeout*0.9), int(timeout*1.1)) + with PreserveLoggingContext(): events, tokens = yield self.notifier.get_events_for( auth_user, room_ids, pagin_config, timeout -- cgit 1.4.1 From 83d31144ebe174b884dd779f1be930bd69bf8e9a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 6 Mar 2015 10:26:08 +0000 Subject: Add the highlight tweak where messages should be highlighted a different colour in appropriate clients. --- synapse/push/baserules.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 91c528cc19..4b4c14816b 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -60,6 +60,8 @@ def make_base_content_rules(user): { 'set_tweak': 'sound', 'value': 'default', + }, { + 'set_tweak': 'highlight' } ] }, @@ -80,6 +82,8 @@ def make_base_override_rules(): { 'set_tweak': 'sound', 'value': 'default' + }, { + 'set_tweak': 'highlight' } ] }, -- cgit 1.4.1 From 1487bba226019d187f38ba8c98b710515cb9bdde Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 6 Mar 2015 10:27:32 +0000 Subject: Suppress notices should trump content/room/sender rules. --- synapse/push/baserules.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 4b4c14816b..fcc6633156 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -70,6 +70,19 @@ def make_base_content_rules(user): def make_base_override_rules(): return [ + { + 'rule_id': 'global/underride/.m.rule.suppress_notices', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'content.msgtype', + 'pattern': 'm.notice', + } + ], + 'actions': [ + 'dont-notify', + ] + }, { 'rule_id': 'global/override/.m.rule.contains_display_name', 'conditions': [ @@ -108,19 +121,6 @@ def make_base_override_rules(): def make_base_underride_rules(): return [ - { - 'rule_id': 'global/underride/.m.rule.suppress_notices', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'content.msgtype', - 'pattern': 'm.notice', - } - ], - 'actions': [ - 'dont-notify', - ] - }, { 'rule_id': 'global/underride/.m.rule.fallback', 'conditions': [ -- cgit 1.4.1 From 3ce8540484a3cc29ce2970ebf6608b6fd3359931 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Mar 2015 11:34:06 +0000 Subject: Don't look for an TLS private key if we have set --no-tls --- synapse/config/server.py | 3 --- synapse/config/tls.py | 17 +++++++++++++---- synapse/crypto/context_factory.py | 5 ++++- 3 files changed, 17 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/config/server.py b/synapse/config/server.py index 4e4892d40b..b042d4eed9 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -30,7 +30,6 @@ class ServerConfig(Config): self.pid_file = self.abspath(args.pid_file) self.webclient = True self.manhole = args.manhole - self.no_tls = args.no_tls self.soft_file_limit = args.soft_file_limit if not args.content_addr: @@ -76,8 +75,6 @@ class ServerConfig(Config): server_group.add_argument("--content-addr", default=None, help="The host and scheme to use for the " "content repository") - server_group.add_argument("--no-tls", action='store_true', - help="Don't bind to the https port.") server_group.add_argument("--soft-file-limit", type=int, default=0, help="Set the soft limit on the number of " "file descriptors synapse can use. " diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 384b29e7ba..a45bf6d521 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import Config +from ._base import Config, ConfigError from OpenSSL import crypto import subprocess @@ -28,9 +28,16 @@ class TlsConfig(Config): self.tls_certificate = self.read_tls_certificate( args.tls_certificate_path ) - self.tls_private_key = self.read_tls_private_key( - args.tls_private_key_path - ) + + self.no_tls = args.no_tls + + if self.no_tls: + self.tls_private_key = None + else: + self.tls_private_key = self.read_tls_private_key( + args.tls_private_key_path + ) + self.tls_dh_params_path = self.check_file( args.tls_dh_params_path, "tls_dh_params" ) @@ -45,6 +52,8 @@ class TlsConfig(Config): help="PEM encoded private key for TLS") tls_group.add_argument("--tls-dh-params-path", help="PEM dh parameters for ephemeral keys") + tls_group.add_argument("--no-tls", action='store_true', + help="Don't bind to the https port.") def read_tls_certificate(self, cert_path): cert_pem = self.read_file(cert_path, "tls_certificate") diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 24d4abf3e9..2f8618a0df 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -38,7 +38,10 @@ class ServerContextFactory(ssl.ContextFactory): logger.exception("Failed to enable eliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) context.use_certificate(config.tls_certificate) - context.use_privatekey(config.tls_private_key) + + if not config.no_tls: + context.use_privatekey(config.tls_private_key) + context.load_tmp_dh(config.tls_dh_params_path) context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH") -- cgit 1.4.1 From 657a0d2568133a06964cbfb12a58c98b259e7f43 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 6 Mar 2015 11:34:30 +0000 Subject: Comment typo --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index ba1aac30fb..086010427f 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -121,7 +121,7 @@ class Pusher(object): ev, c, display_name=my_display_name, room_member_count=room_member_count ) - # ignore rules with no actions (we have an explict 'dont_notify' + # ignore rules with no actions (we have an explict 'dont_notify') if len(actions) == 0: logger.warn( "Ignoring rule id %s with no actions for user %s" % -- cgit 1.4.1 From e49d6b1568eab259dc5eea434da2d0e65876e492 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Mar 2015 11:37:24 +0000 Subject: Unused import --- synapse/config/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/config/tls.py b/synapse/config/tls.py index a45bf6d521..034f9a7bf0 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import Config, ConfigError +from ._base import Config from OpenSSL import crypto import subprocess -- cgit 1.4.1 From c06b45129c218b87867595351e6500d17bc0bd92 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 6 Mar 2015 11:50:51 +0000 Subject: Add more server default rules so we have default rules for whether you get notifs for invites / random member events --- synapse/push/__init__.py | 8 ++--- synapse/push/baserules.py | 74 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 6 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 086010427f..3da0ce8703 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -72,10 +72,6 @@ class Pusher(object): # let's assume you probably know about messages you sent yourself defer.returnValue(['dont_notify']) - if ev['type'] == 'm.room.member': - if ev['state_key'] != self.user_name: - defer.returnValue(['dont_notify']) - rawrules = yield self.store.get_push_rules_for_user(self.user_name) for r in rawrules: @@ -121,6 +117,10 @@ class Pusher(object): ev, c, display_name=my_display_name, room_member_count=room_member_count ) + logger.debug( + "Rule %s %s", + r['rule_id'], "matches" if matches else "doesn't match" + ) # ignore rules with no actions (we have an explict 'dont_notify') if len(actions) == 0: logger.warn( diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index fcc6633156..eddc7fcbe2 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -33,7 +33,7 @@ def make_base_rules(user, kind): if kind == 'override': rules = make_base_override_rules() elif kind == 'underride': - rules = make_base_underride_rules() + rules = make_base_underride_rules(user) elif kind == 'content': rules = make_base_content_rules(user) @@ -119,8 +119,78 @@ def make_base_override_rules(): ] -def make_base_underride_rules(): +def make_base_underride_rules(user): return [ + { + 'rule_id': 'global/underride/.m.rule.invite_for_me', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'type', + 'pattern': 'm.room.member', + }, + { + 'kind': 'event_match', + 'key': 'content.membership', + 'pattern': 'invite', + }, + { + 'kind': 'event_match', + 'key': 'state_key', + 'pattern': user.to_string(), + }, + ], + 'actions': [ + 'notify', + { + 'set_tweak': 'sound', + 'value': 'default' + } + ] + }, + { + 'rule_id': 'global/underride/.m.rule.member_event', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'type', + 'pattern': 'm.room.member', + } + ], + 'actions': [ + 'notify', + ] + }, + { + 'rule_id': 'global/underride/.m.rule.message', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'type', + 'pattern': 'm.room.message', + } + ], + 'actions': [ + 'notify', + ] + }, + { + 'rule_id': 'global/underride/.m.rule.call', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'type', + 'pattern': 'm.call.invite', + } + ], + 'actions': [ + 'notify', + { + 'set_tweak': 'sound', + 'value': 'ring' + } + ] + }, { 'rule_id': 'global/underride/.m.rule.fallback', 'conditions': [ -- cgit 1.4.1 From cf66ddc1b4eb95fc08d8d3f9292c5c7aa899ccd0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 6 Mar 2015 14:11:49 +0000 Subject: Schema change as delta in v14 --- synapse/storage/schema/delta/14/v14.sql | 9 +++++++++ synapse/storage/schema/delta/next_pushrules2.sql | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 synapse/storage/schema/delta/14/v14.sql delete mode 100644 synapse/storage/schema/delta/next_pushrules2.sql (limited to 'synapse') diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql new file mode 100644 index 0000000000..0212726448 --- /dev/null +++ b/synapse/storage/schema/delta/14/v14.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS push_rules_enable ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + enabled TINYINT, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/delta/next_pushrules2.sql b/synapse/storage/schema/delta/next_pushrules2.sql deleted file mode 100644 index 0212726448..0000000000 --- a/synapse/storage/schema/delta/next_pushrules2.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE IF NOT EXISTS push_rules_enable ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, - enabled TINYINT, - UNIQUE(user_name, rule_id) -); - -CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); -- cgit 1.4.1 From 369449827d3b6176c6ada812f403f519357a740e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Mar 2015 14:24:53 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 74c5584a1a..f46a6df1fb 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.1-r4" +__version__ = "0.8.0" -- cgit 1.4.1 From 5ecc768970c7f9c0184c48df12c90ffb2042b270 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 6 Mar 2015 14:41:50 +0000 Subject: Add attribute so push gateways can tell if a member event is about the user in question --- synapse/push/httppusher.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 202fcb42f4..caca709f10 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -88,6 +88,7 @@ class HttpPusher(Pusher): } if event['type'] == 'm.room.member': d['notification']['membership'] = event['content']['membership'] + d['notification']['user_is_target'] = event['state_key'] == self.user_name if 'content' in event: d['notification']['content'] = event['content'] -- cgit 1.4.1 From e7d7152c3c88467e045f7c39e60e3ffeab40139b Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 6 Mar 2015 15:03:34 +0000 Subject: Remove the fallback rule - we probably don't want to be notifying for everything even if we don't know what it is. --- synapse/push/baserules.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index eddc7fcbe2..0c91e77c62 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -190,13 +190,5 @@ def make_base_underride_rules(user): 'value': 'ring' } ] - }, - { - 'rule_id': 'global/underride/.m.rule.fallback', - 'conditions': [ - ], - 'actions': [ - 'notify', - ] - }, + } ] -- cgit 1.4.1 From 96eda876a4e52269b7eec62ccdd4698aa1721b8e Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 6 Mar 2015 15:12:37 +0000 Subject: Specify when we don't want to highlight --- synapse/push/baserules.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 0c91e77c62..6d00ebd36d 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -113,6 +113,9 @@ def make_base_override_rules(): { 'set_tweak': 'sound', 'value': 'default' + }, { + 'set_tweak': 'highlight', + 'value': 'false' } ] } @@ -145,6 +148,9 @@ def make_base_underride_rules(user): { 'set_tweak': 'sound', 'value': 'default' + }, { + 'set_tweak': 'highlight', + 'value': 'false' } ] }, @@ -158,7 +164,10 @@ def make_base_underride_rules(user): } ], 'actions': [ - 'notify', + 'notify', { + 'set_tweak': 'highlight', + 'value': 'false' + } ] }, { @@ -171,7 +180,10 @@ def make_base_underride_rules(user): } ], 'actions': [ - 'notify', + 'notify', { + 'set_tweak': 'highlight', + 'value': 'false' + } ] }, { @@ -188,6 +200,9 @@ def make_base_underride_rules(user): { 'set_tweak': 'sound', 'value': 'ring' + }, { + 'set_tweak': 'highlight', + 'value': 'false' } ] } -- cgit 1.4.1 From be9dafcd37fd3db1b4e0bccf9417d9ddfe344958 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 6 Mar 2015 15:32:38 +0000 Subject: Dial down logging for failed pushers --- synapse/push/httppusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index caca709f10..a02fed57b4 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -109,7 +109,7 @@ class HttpPusher(Pusher): try: resp = yield self.httpCli.post_json_get_json(self.url, notification_dict) except: - logger.exception("Failed to push %s ", self.url) + logger.warn("Failed to push %s ", self.url) defer.returnValue(False) rejected = [] if 'rejected' in resp: -- cgit 1.4.1 From 9ccccd4874fe827baa2583829602663d1936b0ae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Mar 2015 16:24:05 +0000 Subject: When setting display name more graciously handle failures to update room state. --- synapse/handlers/profile.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 03b2159c53..2ddf9d5378 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -212,10 +212,16 @@ class ProfileHandler(BaseHandler): ) msg_handler = self.hs.get_handlers().message_handler - yield msg_handler.create_and_send_event({ - "type": EventTypes.Member, - "room_id": j.room_id, - "state_key": user.to_string(), - "content": content, - "sender": user.to_string() - }, ratelimit=False) + try: + yield msg_handler.create_and_send_event({ + "type": EventTypes.Member, + "room_id": j.room_id, + "state_key": user.to_string(), + "content": content, + "sender": user.to_string() + }, ratelimit=False) + except Exception as e: + logger.warn( + "Failed to update join event for room %s - %s", + j.room_id, str(e.message) + ) -- cgit 1.4.1 From 118c883429d1564f63ab37424a8e0bc602c10709 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 6 Mar 2015 19:36:12 +0000 Subject: Call notifications should be override else they'll get clobbered by sender/room rules. --- synapse/push/baserules.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 6d00ebd36d..b33714f1f6 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -71,7 +71,27 @@ def make_base_content_rules(user): def make_base_override_rules(): return [ { - 'rule_id': 'global/underride/.m.rule.suppress_notices', + 'rule_id': 'global/override/.m.rule.call', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'type', + 'pattern': 'm.call.invite', + } + ], + 'actions': [ + 'notify', + { + 'set_tweak': 'sound', + 'value': 'ring' + }, { + 'set_tweak': 'highlight', + 'value': 'false' + } + ] + }, + { + 'rule_id': 'global/override/.m.rule.suppress_notices', 'conditions': [ { 'kind': 'event_match', @@ -185,25 +205,5 @@ def make_base_underride_rules(user): 'value': 'false' } ] - }, - { - 'rule_id': 'global/underride/.m.rule.call', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'type', - 'pattern': 'm.call.invite', - } - ], - 'actions': [ - 'notify', - { - 'set_tweak': 'sound', - 'value': 'ring' - }, { - 'set_tweak': 'highlight', - 'value': 'false' - } - ] } ] -- cgit 1.4.1 From 1df3ccf7ee646690570ce1d799b3c6c226179e3d Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 9 Mar 2015 12:39:56 +0000 Subject: D'oh: underscore, not hyphen --- synapse/push/baserules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index b33714f1f6..6e333a3d21 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -100,7 +100,7 @@ def make_base_override_rules(): } ], 'actions': [ - 'dont-notify', + 'dont_notify', ] }, { -- cgit 1.4.1