diff options
| author | David Baker <dave@matrix.org> | 2015-01-13 13:15:51 +0000 |
|---|---|---|
| committer | David Baker <dave@matrix.org> | 2015-01-13 13:15:51 +0000 |
| commit | c06a9063e1d838f776edfd79cfc8ab29c748d794 (patch) | |
| tree | bcfa472e65d4dacbab666d5787eff9293e5ccc41 | |
| parent | Split out function to decide whether to notify or a given event (diff) | |
| parent | Merge branch 'hotfixes-v0.6.1b' of github.com:matrix-org/synapse into develop (diff) | |
| download | synapse-c06a9063e1d838f776edfd79cfc8ab29c748d794.tar.xz | |
Merge branch 'develop' into pushers
150 files changed, 1097 insertions, 438 deletions
diff --git a/CHANGES.rst b/CHANGES.rst index 23bdac6a8a..297ae914fd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,21 @@ +Changes in synapse 0.6.1 (2015-01-07) +===================================== + + * Major optimizations to improve performance of initial sync and event sending + in large rooms (by up to 10x) + * Media repository now includes a Content-Length header on media downloads. + * Improve quality of thumbnails by changing resizing algorithm. + Changes in synapse 0.6.0 (2014-12-16) ===================================== * Add new API for media upload and download that supports thumbnailing. + * Replicate media uploads over multiple homeservers so media is always served + to clients from their local homeserver. This obsoletes the + --content-addr parameter and confusion over accessing content directly + from remote homeservers. + * Implement exponential backoff when retrying federation requests when + sending to remote homeservers which are offline. * Implement typing notifications. * Fix bugs where we sent events with invalid signatures due to bugs where we incorrectly persisted events. diff --git a/README.rst b/README.rst index f5d2b0af38..768da3df64 100644 --- a/README.rst +++ b/README.rst @@ -108,6 +108,18 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into ``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX. +Your python may not give priority to locally installed libraries over system +libraries, in which case you must add your local packages to your python path:: + + $ # on Linux: + $ export PYTHONPATH=$HOME/.local/lib/python2.7/site-packages:$PYTHONPATH + + $ # on OSX: + $ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python/site-packages:$PYTHONPATH + +For reliable VoIP calls to be routed via this homeserver, you MUST configure +a TURN server. See docs/turn-howto.rst for details. + Troubleshooting Installation ---------------------------- @@ -239,6 +251,11 @@ Upgrading an existing homeserver IMPORTANT: Before upgrading an existing homeserver to a new version, please refer to UPGRADE.rst for any additional instructions. +Otherwise, simply re-install the new codebase over the current one - e.g. +by ``pip install --user --process-dependency-links +https://github.com/matrix-org/synapse/tarball/master`` +if using pip, or by ``git pull`` if running off a git working copy. + Setting up Federation ===================== diff --git a/UPGRADE.rst b/UPGRADE.rst index a602a9f3eb..9618ad2d57 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,6 +1,10 @@ Upgrading to v0.6.0 =================== +To pull in new dependencies, run:: + + python setup.py develop --user + This update includes a change to the database schema. To upgrade you first need to upgrade the database by running:: diff --git a/VERSION b/VERSION index a918a2aa18..3b3e723172 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.0 +0.6.1b diff --git a/docs/media_repository.rst b/docs/media_repository.rst index e4a6974041..1037b5be63 100644 --- a/docs/media_repository.rst +++ b/docs/media_repository.rst @@ -1,6 +1,8 @@ -Media Repository +Media Repository ================ +*Synapse implementation-specific details for the media repository* + The media repository is where attachments and avatar photos are stored. It stores attachment content and thumbnails for media uploaded by local users. It caches attachment content and thumbnails for media uploaded by remote users. diff --git a/graph/graph2.py b/graph/graph2.py index b9b8a562a0..6b551d42e5 100644 --- a/graph/graph2.py +++ b/graph/graph2.py @@ -23,14 +23,27 @@ import argparse from synapse.events import FrozenEvent -def make_graph(db_name, room_id, file_prefix): +def make_graph(db_name, room_id, file_prefix, limit): conn = sqlite3.connect(db_name) - c = conn.execute( - "SELECT json FROM event_json where room_id = ?", - (room_id,) + sql = ( + "SELECT json FROM event_json as j " + "INNER JOIN events as e ON e.event_id = j.event_id " + "WHERE j.room_id = ?" ) + args = [room_id] + + if limit: + sql += ( + " ORDER BY topological_ordering DESC, stream_ordering DESC " + "LIMIT ?" + ) + + args.append(limit) + + c = conn.execute(sql, args) + events = [FrozenEvent(json.loads(e[0])) for e in c.fetchall()] events.sort(key=lambda e: e.depth) @@ -128,11 +141,16 @@ if __name__ == "__main__": ) parser.add_argument( "-p", "--prefix", dest="prefix", - help="String to prefix output files with" + help="String to prefix output files with", + default="graph_output" + ) + parser.add_argument( + "-l", "--limit", + help="Only retrieve the last N events.", ) parser.add_argument('db') parser.add_argument('room') args = parser.parse_args() - make_graph(args.db, args.room, args.prefix) + make_graph(args.db, args.room, args.prefix, args.limit) diff --git a/scripts/copyrighter-sql.pl b/scripts/copyrighter-sql.pl new file mode 100755 index 0000000000..890e51e587 --- /dev/null +++ b/scripts/copyrighter-sql.pl @@ -0,0 +1,33 @@ +#!/usr/bin/perl -pi +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +$copyright = <<EOT; +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +EOT + +s/^(# -\*- coding: utf-8 -\*-\n)?/$1$copyright/ if ($. == 1); diff --git a/scripts/copyrighter.pl b/scripts/copyrighter.pl index 7c03ef21fc..a913d74c8d 100755 --- a/scripts/copyrighter.pl +++ b/scripts/copyrighter.pl @@ -14,7 +14,7 @@ # limitations under the License. $copyright = <<EOT; -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/setup.py b/setup.py index eb410aa6d3..043cd044a7 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "matrix_angular_sdk==0.5.3b", + "matrix_angular_sdk==0.6.0", "Twisted>=14.0.0", "service_identity>=1.0.0", "pyopenssl>=0.14", @@ -47,7 +47,7 @@ setup( dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.3b/#egg=matrix_angular_sdk-0.5.3b", + "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.0", ], setup_requires=[ "setuptools_trial", diff --git a/synapse/__init__.py b/synapse/__init__.py index 1cef40925f..895a0766d2 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" This is a reference implementation of a synapse home server. +""" This is a reference implementation of a Matrix home server. """ -__version__ = "0.6.0" +__version__ = "0.6.1b" diff --git a/synapse/api/__init__.py b/synapse/api/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/api/__init__.py +++ b/synapse/api/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 70245aba04..e31482cfaa 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 4fc8b79a40..7ee6dcc46e 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 5cda3a6953..a4155aebae 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index b25358090f..3f9ad4ce89 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/urls.py b/synapse/api/urls.py index d7625127f8..a299392049 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ea52259724..3d85fda67b 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ from synapse.storage import prepare_database, UpgradeDatabaseException from synapse.server import HomeServer +from synapse.python_dependencies import check_requirements + from twisted.internet import reactor from twisted.enterprise import adbapi from twisted.web.resource import Resource @@ -39,6 +41,8 @@ from synapse.util.logcontext import LoggingContext from daemonize import Daemonize import twisted.manhole.telnet +import synapse + import logging import os import re @@ -198,7 +202,10 @@ def setup(): config.setup_logging() + check_requirements() + logger.info("Server hostname: %s", config.server_name) + logger.info("Server version: %s", synapse.__version__) if re.search(":[0-9]+$", config.server_name): domain_with_port = config.server_name @@ -234,7 +241,7 @@ def setup(): except UpgradeDatabaseException: sys.stderr.write( "\nFailed to upgrade database.\n" - "Have you followed any instructions in UPGRADES.rst?\n" + "Have you checked for version specific instructions in UPGRADES.rst?\n" ) sys.exit(1) @@ -279,6 +286,7 @@ def run(): def main(): with LoggingContext("main"): + check_requirements() setup() diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 52a0b729f4..363c20f994 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/__init__.py b/synapse/config/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/config/__init__.py +++ b/synapse/config/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 1cdd03e414..dfc115d8e8 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 4ed9070b9e..7e21c7414d 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/database.py b/synapse/config/database.py index 0aac8c8382..0d33583a7d 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/email.py b/synapse/config/email.py index 6bab133224..f0854f8c37 100644 --- a/synapse/config/email.py +++ b/synapse/config/email.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 5a11fd6c76..b0fe217459 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/logger.py b/synapse/config/logger.py index d352ea9be9..f9568ebd21 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -66,7 +66,10 @@ class LoggingConfig(Config): formatter = logging.Formatter(log_format) if self.log_file: - handler = logging.FileHandler(self.log_file) + # TODO: Customisable file size / backup count + handler = logging.handlers.RotatingFileHandler( + self.log_file, maxBytes=(1000 * 1000 * 100), backupCount=3 + ) else: handler = logging.StreamHandler() handler.setFormatter(formatter) diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index fb63ed7d9b..17c7e64ce7 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/repository.py b/synapse/config/repository.py index f1b7b1b74e..e1827f05e4 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014, 2015 matrix.org # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ class ContentRepositoryConfig(Config): super(ContentRepositoryConfig, cls).add_arguments(parser) db_group = parser.add_argument_group("content_repository") db_group.add_argument( - "--max-upload-size", default="1M" + "--max-upload-size", default="10M" ) db_group.add_argument( "--media-store-path", default=cls.default_path("media_store") diff --git a/synapse/config/server.py b/synapse/config/server.py index f8a0844b8c..31e44cc857 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -47,8 +47,12 @@ class ServerConfig(Config): def add_arguments(cls, parser): super(ServerConfig, cls).add_arguments(parser) server_group = parser.add_argument_group("server") - server_group.add_argument("-H", "--server-name", default="localhost", - help="The name of the server") + server_group.add_argument( + "-H", "--server-name", default="localhost", + help="The domain name of the server, with optional explicit port. " + "This is used by remote servers to connect to this server, " + "e.g. matrix.org, localhost:8080, etc." + ) server_group.add_argument("--signing-key-path", help="The signing key to sign messages with") server_group.add_argument("-p", "--bind-port", metavar="PORT", diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 3600c3ea9e..384b29e7ba 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/voip.py b/synapse/config/voip.py index 06675966ce..a2b822719f 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/__init__.py b/synapse/crypto/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/crypto/__init__.py +++ b/synapse/crypto/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 3143322d9c..24d4abf3e9 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -1,4 +1,4 @@ -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 21c19c971d..6633b19565 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 3f37c99261..9c910fa3fc 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index ceb03ce6c2..3fb99f7125 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 34b1b944ab..4252e5ab5c 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,12 +15,10 @@ from synapse.util.frozenutils import freeze, unfreeze -import copy - class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): - self.__dict__ = copy.deepcopy(internal_metadata_dict) + self.__dict__ = internal_metadata_dict def get_dict(self): return dict(self.__dict__) @@ -49,10 +47,10 @@ def _event_dict_property(key): class EventBase(object): def __init__(self, event_dict, signatures={}, unsigned={}, internal_metadata_dict={}): - self.signatures = copy.deepcopy(signatures) - self.unsigned = copy.deepcopy(unsigned) + self.signatures = signatures + self.unsigned = unsigned - self._event_dict = copy.deepcopy(event_dict) + self._event_dict = event_dict self.internal_metadata = _EventInternalMetadata( internal_metadata_dict @@ -112,10 +110,16 @@ class EventBase(object): class FrozenEvent(EventBase): def __init__(self, event_dict, internal_metadata_dict={}): - event_dict = copy.deepcopy(event_dict) + event_dict = dict(event_dict) + + # Signatures is a dict of dicts, and this is faster than doing a + # copy.deepcopy + signatures = { + name: {sig_id: sig for sig_id, sig in sigs.items()} + for name, sigs in event_dict.pop("signatures", {}).items() + } - signatures = copy.deepcopy(event_dict.pop("signatures", {})) - unsigned = copy.deepcopy(event_dict.pop("unsigned", {})) + unsigned = dict(event_dict.pop("unsigned", {})) frozen_dict = freeze(event_dict) diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 9579b1fe8b..d4cb602ebb 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index b9fb29be01..6bbba8d6ba 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 94f3f15f52..bcb5457278 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -89,16 +89,24 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(hs, e): +def serialize_event(hs, e, client_event=True): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e # Should this strip out None's? d = {k: v for k, v in e.get_dict().items()} + + if not client_event: + # set the age and keep all other keys + if "age_ts" in d["unsigned"]: + now = int(hs.get_clock().time_msec()) + d["unsigned"]["age"] = now - d["unsigned"]["age_ts"] + return d + if "age_ts" in d["unsigned"]: now = int(hs.get_clock().time_msec()) - d["unsigned"]["age"] = now - d["unsigned"]["age_ts"] + d["age"] = now - d["unsigned"]["age_ts"] del d["unsigned"]["age_ts"] d["user_id"] = d.pop("sender", None) @@ -126,5 +134,8 @@ def serialize_event(hs, e): del d["prev_events"] del d["hashes"] del d["signatures"] + d.pop("depth", None) + d.pop("unsigned", None) + d.pop("origin", None) return d diff --git a/synapse/events/validator.py b/synapse/events/validator.py index ebc6c30e62..0ee6872d1e 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/__init__.py b/synapse/federation/__init__.py index 0112588656..7517c529d4 100644 --- a/synapse/federation/__init__.py +++ b/synapse/federation/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 73dc844d59..85c82a4623 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9f8aadccca..a4c29b484b 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -256,31 +256,35 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def get_state_for_context(self, destination, context, event_id=None): + def get_state_for_context(self, destination, context, event_id): """Requests all of the `current` state PDUs for a given context from a remote home server. Args: destination (str): The remote homeserver to query for the state. context (str): The context we're interested in. + event_id (str): The id of the event we want the state at. Returns: Deferred: Results in a list of PDUs. """ - transaction_data = yield self.transport_layer.get_context_state( + result = yield self.transport_layer.get_context_state( destination, context, event_id=event_id, ) - transaction = Transaction(**transaction_data) pdus = [ + self.event_from_pdu_json(p, outlier=True) for p in result["pdus"] + ] + + auth_chain = [ self.event_from_pdu_json(p, outlier=True) - for p in transaction.pdus + for p in result.get("auth_chain", []) ] - defer.returnValue(pdus) + defer.returnValue((pdus, auth_chain)) @defer.inlineCallbacks @log_function @@ -383,10 +387,16 @@ class ReplicationLayer(object): context, event_id, ) + auth_chain = yield self.store.get_auth_chain( + [pdu.event_id for pdu in pdus] + ) else: raise NotImplementedError("Specify an event") - defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) + defer.returnValue((200, { + "pdus": [pdu.get_pdu_json() for pdu in pdus], + "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain], + })) @defer.inlineCallbacks @log_function @@ -562,8 +572,8 @@ class ReplicationLayer(object): already_seen = ( existing and ( - not existing.internal_metadata.outlier - or pdu.internal_metadata.outlier + not existing.internal_metadata.is_outlier() + or pdu.internal_metadata.is_outlier() ) ) if already_seen: @@ -573,6 +583,8 @@ class ReplicationLayer(object): state = None + auth_chain = [] + # We need to make sure we have all the auth events. # for e_id, _ in pdu.auth_events: # exists = yield self._get_persisted_pdu( @@ -604,7 +616,7 @@ class ReplicationLayer(object): # ) # Get missing pdus if necessary. - if not pdu.internal_metadata.outlier: + if not pdu.internal_metadata.is_outlier(): # We only backfill backwards to the min depth. min_depth = yield self.handler.get_min_depth_for_context( pdu.room_id @@ -645,7 +657,7 @@ class ReplicationLayer(object): "_handle_new_pdu getting state for %s", pdu.room_id ) - state = yield self.get_state_for_context( + state, auth_chain = yield self.get_state_for_context( origin, pdu.room_id, pdu.event_id, ) @@ -655,6 +667,7 @@ class ReplicationLayer(object): pdu, backfilled=backfilled, state=state, + auth_chain=auth_chain, ) else: ret = None @@ -717,6 +730,7 @@ class _TransactionQueue(object): destinations = set(destinations) destinations.discard(self.server_name) + destinations.discard("localhost") logger.debug("Sending to: %s", str(destinations)) @@ -801,6 +815,8 @@ class _TransactionQueue(object): else: logger.info("TX [%s] is ready for retry", destination) + logger.info("TX [%s] _attempt_new_transaction", destination) + if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending # request at which point pending_pdus_by_dest just keeps growing. @@ -813,6 +829,9 @@ class _TransactionQueue(object): pending_edus = self.pending_edus_by_dest.pop(destination, []) pending_failures = self.pending_failures_by_dest.pop(destination, []) + if pending_pdus: + logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", destination, len(pending_pdus)) + if not pending_pdus and not pending_edus and not pending_failures: return diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 0f11c6d491..1f0f06e0fe 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 1bcd0548c2..816f55bf39 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index d5df3c630b..fe071a4bc2 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 97ebd98917..38af034b4d 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 687b343a1d..1c9e7152c7 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 66d3b533d9..91fceda2ac 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -155,4 +155,4 @@ class DirectoryHandler(BaseHandler): "room_id": room_id, "sender": user_id, "content": {"aliases": aliases}, - }) + }, ratelimit=False) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 02202692d4..c9ade253dd 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -46,7 +46,8 @@ class EventStreamHandler(BaseHandler): @defer.inlineCallbacks @log_function - def get_stream(self, auth_user_id, pagin_config, timeout=0): + def get_stream(self, auth_user_id, pagin_config, timeout=0, + as_client_event=True): auth_user = self.hs.parse_userid(auth_user_id) try: @@ -78,7 +79,9 @@ class EventStreamHandler(BaseHandler): auth_user, room_ids, pagin_config, timeout ) - chunks = [self.hs.serialize_event(e) for e in events] + chunks = [ + self.hs.serialize_event(e, as_client_event) for e in events + ] chunk = { "chunk": chunks, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4aec3563ac..195f7c618a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -91,11 +91,12 @@ class FederationHandler(BaseHandler): yield run_on_reactor() - yield self.replication_layer.send_pdu(event, destinations) + self.replication_layer.send_pdu(event, destinations) @log_function @defer.inlineCallbacks - def on_receive_pdu(self, origin, pdu, backfilled, state=None): + def on_receive_pdu(self, origin, pdu, backfilled, state=None, + auth_chain=None): """ Called by the ReplicationLayer when we have a new pdu. We need to do auth checks and put it through the StateHandler. """ @@ -150,40 +151,41 @@ class FederationHandler(BaseHandler): if not is_in_room and not event.internal_metadata.outlier: logger.debug("Got event for room we're not in.") - replication_layer = self.replication_layer - auth_chain = yield replication_layer.get_event_auth( - origin, - context=event.room_id, - event_id=event.event_id, - ) + replication = self.replication_layer + + if not state: + state, auth_chain = yield replication.get_state_for_context( + origin, context=event.room_id, event_id=event.event_id, + ) + + if not auth_chain: + auth_chain = yield replication.get_event_auth( + origin, + context=event.room_id, + event_id=event.event_id, + ) for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e, fetch_missing=False) + yield self._handle_new_event(e, fetch_auth_from=origin) except: logger.exception( - "Failed to parse auth event %s", + "Failed to handle auth event %s", e.event_id, ) - if not state: - state = yield replication_layer.get_state_for_context( - origin, - context=event.room_id, - event_id=event.event_id, - ) - current_state = state if state: for e in state: + logging.info("A :) %r", e) e.internal_metadata.outlier = True try: yield self._handle_new_event(e) except: logger.exception( - "Failed to parse state event %s", + "Failed to handle state event %s", e.event_id, ) @@ -288,7 +290,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def on_event_auth(self, event_id): - auth = yield self.store.get_auth_chain(event_id) + auth = yield self.store.get_auth_chain([event_id]) for event in auth: event.signatures.update( @@ -391,10 +393,10 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e, fetch_missing=False) + yield self._handle_new_event(e) except: logger.exception( - "Failed to parse auth event %s", + "Failed to handle auth event %s", e.event_id, ) @@ -403,12 +405,11 @@ class FederationHandler(BaseHandler): e.internal_metadata.outlier = True try: yield self._handle_new_event( - e, - fetch_missing=True + e, fetch_auth_from=target_host ) except: logger.exception( - "Failed to parse state event %s", + "Failed to handle state event %s", e.event_id, ) @@ -526,9 +527,12 @@ class FederationHandler(BaseHandler): event.signatures, ) - yield self.replication_layer.send_pdu(new_pdu, destinations) + self.replication_layer.send_pdu(new_pdu, destinations) - auth_chain = yield self.store.get_auth_chain(event.event_id) + state_ids = [e.event_id for e in context.current_state.values()] + auth_chain = yield self.store.get_auth_chain(set( + [event.event_id] + state_ids + )) defer.returnValue({ "state": context.current_state.values(), @@ -613,13 +617,13 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function - def on_backfill_request(self, origin, context, pdu_list, limit): - in_room = yield self.auth.check_host_in_room(context, origin) + def on_backfill_request(self, origin, room_id, pdu_list, limit): + in_room = yield self.auth.check_host_in_room(room_id, origin) if not in_room: raise AuthError(403, "Host not in room.") events = yield self.store.get_backfill_events( - context, + room_id, pdu_list, limit ) @@ -678,7 +682,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _handle_new_event(self, event, state=None, backfilled=False, - current_state=None, fetch_missing=True): + current_state=None, fetch_auth_from=None): logger.debug( "_handle_new_event: Before annotate: %s, sigs: %s", @@ -699,11 +703,20 @@ class FederationHandler(BaseHandler): known_ids = set( [s.event_id for s in context.auth_events.values()] ) + for e_id, _ in event.auth_events: if e_id not in known_ids: - e = yield self.store.get_event( - e_id, allow_none=True, - ) + e = yield self.store.get_event(e_id, allow_none=True) + + if not e and fetch_auth_from is not None: + # Grab the auth_chain over federation if we are missing + # auth events. + auth_chain = yield self.replication_layer.get_event_auth( + fetch_auth_from, event.event_id, event.room_id + ) + for auth_event in auth_chain: + yield self._handle_new_event(auth_event) + e = yield self.store.get_event(e_id, allow_none=True) if not e: # TODO: Do some conflict res to make sure that we're @@ -713,7 +726,7 @@ class FederationHandler(BaseHandler): event.event_id, e_id, known_ids, ) # FIXME: How does raising AuthError work with federation? - raise AuthError(403, "Auth events are stale") + raise AuthError(403, "Cannot find auth event") context.auth_events[(e.type, e.state_key)] = e diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index c98ae2cfb5..d297d71c03 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 49c0e98113..f2a2f16933 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig from synapse.events.validator import EventValidator +from synapse.util.logcontext import PreserveLoggingContext from ._base import BaseHandler @@ -66,7 +67,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def get_messages(self, user_id=None, room_id=None, pagin_config=None, - feedback=False): + feedback=False, as_client_event=True): """Get messages in a room. Args: @@ -75,6 +76,7 @@ class MessageHandler(BaseHandler): pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. feedback (bool): True to get compressed feedback with the messages + as_client_event (bool): True to get events in client-server format. Returns: dict: Pagination API results """ @@ -98,7 +100,9 @@ class MessageHandler(BaseHandler): ) chunk = { - "chunk": [self.hs.serialize_event(e) for e in events], + "chunk": [ + self.hs.serialize_event(e, as_client_event) for e in events + ], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } @@ -106,7 +110,7 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def create_and_send_event(self, event_dict): + def create_and_send_event(self, event_dict, ratelimit=True): """ Given a dict from a client, create and handle a new event. Creates an FrozenEvent object, filling out auth_events, prev_events, @@ -123,7 +127,8 @@ class MessageHandler(BaseHandler): self.validator.validate_new(builder) - self.ratelimit(builder.user_id) + if ratelimit: + self.ratelimit(builder.user_id) # TODO(paul): Why does 'event' not have a 'user' object? user = self.hs.parse_userid(builder.user_id) assert self.hs.is_mine(user), "User must be our own: %s" % (user,) @@ -152,6 +157,11 @@ class MessageHandler(BaseHandler): context=context, ) + if event.type == EventTypes.Message: + presence = self.hs.get_handlers().presence_handler + with PreserveLoggingContext(): + presence.bump_presence_active_time(user) + defer.returnValue(event) @defer.inlineCallbacks @@ -204,7 +214,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, - feedback=False): + feedback=False, as_client_event=True): """Retrieve a snapshot of all rooms the user is invited or has joined. This snapshot may include messages for all rooms where the user is @@ -215,6 +225,7 @@ class MessageHandler(BaseHandler): pagin_config (synapse.api.streams.PaginationConfig): The pagination config used to determine how many messages *PER ROOM* to return. feedback (bool): True to get feedback along with these messages. + as_client_event (bool): True to get events in client-server format. Returns: A list of dicts with "room_id" and "membership" keys for all rooms the user is currently invited or joined in on. Rooms where the user @@ -256,7 +267,7 @@ class MessageHandler(BaseHandler): } if event.membership == Membership.INVITE: - d["inviter"] = event.user_id + d["inviter"] = event.sender rooms_ret.append(d) @@ -273,7 +284,10 @@ class MessageHandler(BaseHandler): end_token = now_token.copy_and_replace("room_key", token[1]) d["messages"] = { - "chunk": [self.hs.serialize_event(m) for m in messages], + "chunk": [ + self.hs.serialize_event(m, as_client_event) + for m in messages + ], "start": start_token.to_string(), "end": end_token.to_string(), } diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index f3abfecdee..8aeed99274 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 33a2c167ec..7777d3cc94 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException -from synapse.api.constants import Membership +from synapse.api.constants import EventTypes, Membership from synapse.util.logcontext import PreserveLoggingContext from ._base import BaseHandler @@ -194,6 +194,8 @@ class ProfileHandler(BaseHandler): if not self.hs.is_mine(user): return + self.ratelimit(user.to_string()) + joins = yield self.store.get_rooms_for_user_where_membership_is( user.to_string(), [Membership.JOIN], @@ -201,7 +203,7 @@ class ProfileHandler(BaseHandler): for j in joins: content = { - "membership": j.content["membership"], + "membership": Membership.JOIN, } yield self.distributor.fire( @@ -210,9 +212,9 @@ class ProfileHandler(BaseHandler): msg_handler = self.hs.get_handlers().message_handler yield msg_handler.create_and_send_event({ - "type": j.type, + "type": EventTypes.Member, "room_id": j.room_id, - "state_key": j.state_key, + "state_key": user.to_string(), "content": content, - "sender": j.state_key, - }) + "sender": user.to_string() + }, ratelimit=False) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 15d8716455..732652c228 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 8567d7409d..6d0db18e51 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -130,6 +130,7 @@ class RoomCreationHandler(BaseHandler): "type": EventTypes.Name, "room_id": room_id, "sender": user_id, + "state_key": "", "content": {"name": name}, }) @@ -139,6 +140,7 @@ class RoomCreationHandler(BaseHandler): "type": EventTypes.Topic, "room_id": room_id, "sender": user_id, + "state_key": "", "content": {"topic": topic}, }) @@ -147,7 +149,7 @@ class RoomCreationHandler(BaseHandler): "type": EventTypes.Member, "state_key": invitee, "room_id": room_id, - "user_id": user_id, + "sender": user_id, "content": {"membership": Membership.INVITE}, }) @@ -243,14 +245,12 @@ class RoomMemberHandler(BaseHandler): self.distributor.declare("user_left_room") @defer.inlineCallbacks - def get_room_members(self, room_id, membership=Membership.JOIN): + def get_room_members(self, room_id): hs = self.hs - memberships = yield self.store.get_room_members( - room_id=room_id, membership=membership - ) + users = yield self.store.get_users_in_room(room_id) - defer.returnValue([hs.parse_userid(m.user_id) for m in memberships]) + defer.returnValue([hs.parse_userid(u) for u in users]) @defer.inlineCallbacks def fetch_room_distributions_into(self, room_id, localusers=None, @@ -390,6 +390,11 @@ class RoomMemberHandler(BaseHandler): host = hosts[0] + # If event doesn't include a display name, add one. + yield self.distributor.fire( + "collect_presencelike_data", joinee, content + ) + content.update({"membership": Membership.JOIN}) builder = self.event_builder_factory.new({ "type": EventTypes.Member, @@ -420,10 +425,22 @@ class RoomMemberHandler(BaseHandler): event.room_id, self.hs.hostname ) + if not is_host_in_room: + # is *anyone* in the room? + room_member_keys = [ + v for (k, v) in context.current_state.keys() if ( + k == "m.room.member" + ) + ] + if len(room_member_keys) == 0: + # has the room been created so we can join it? + create_event = context.current_state.get(("m.room.create", "")) + if create_event: + is_host_in_room = True if is_host_in_room: should_do_dance = False - elif room_host: + elif room_host: # TODO: Shouldn't this be remote_room_host? should_do_dance = True else: # TODO(markjh): get prev_state from snapshot @@ -437,7 +454,8 @@ class RoomMemberHandler(BaseHandler): should_do_dance = not self.hs.is_mine(inviter) room_host = inviter.domain else: - should_do_dance = False + # return the same error as join_room_alias does + raise SynapseError(404, "No known servers") if should_do_dance: handler = self.hs.get_handlers().federation_handler @@ -524,11 +542,10 @@ class RoomListHandler(BaseHandler): def get_public_room_list(self): chunk = yield self.store.get_rooms(is_public=True) for room in chunk: - joined_members = yield self.store.get_room_members( + joined_users = yield self.store.get_users_in_room( room_id=room["room_id"], - membership=Membership.JOIN ) - room["num_joined_members"] = len(joined_members) + room["num_joined_members"] = len(joined_users) # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 7626b07280..22ce7873d0 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -83,9 +83,15 @@ class TypingNotificationHandler(BaseHandler): if member in self._member_typing_timer: self.clock.cancel_call_later(self._member_typing_timer[member]) + def _cb(): + logger.debug( + "%s has timed out in %s", target_user.to_string(), room_id + ) + self._stopped_typing(member) + self._member_typing_until[member] = until self._member_typing_timer[member] = self.clock.call_later( - timeout / 1000, lambda: self._stopped_typing(member) + timeout / 1000.0, _cb ) if was_present: diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/http/__init__.py +++ b/synapse/http/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/agent_name.py b/synapse/http/agent_name.py new file mode 100644 index 0000000000..d761890863 --- /dev/null +++ b/synapse/http/agent_name.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse import __version__ + +AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii") diff --git a/synapse/http/client.py b/synapse/http/client.py index 82e80385ce..198f575cfa 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ # limitations under the License. +from synapse.http.agent_name import AGENT_NAME from twisted.internet import defer, reactor from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError @@ -51,7 +52,8 @@ class SimpleHttpClient(object): "POST", uri.encode("ascii"), headers=Headers({ - "Content-Type": ["application/x-www-form-urlencoded"] + b"Content-Type": [b"application/x-www-form-urlencoded"], + b"User-Agent": [AGENT_NAME], }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -105,6 +107,9 @@ class SimpleHttpClient(object): response = yield self.agent.request( "GET", uri.encode("ascii"), + headers=Headers({ + b"User-Agent": [AGENT_NAME], + }) ) body = yield readBody(response) @@ -127,7 +132,8 @@ class CaptchaServerHttpClient(SimpleHttpClient): url.encode("ascii"), bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ - "Content-Type": ["application/x-www-form-urlencoded"] + b"Content-Type": [b"application/x-www-form-urlencoded"], + b"User-Agent": [AGENT_NAME], }) ) diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index 9c8888f565..4ae45f136d 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 8f4db59c75..aa14782b0f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone +from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext @@ -71,6 +72,7 @@ class MatrixFederationHttpClient(object): requests. """ + def __init__(self, hs): self.hs = hs self.signing_key = hs.config.signing_key[0] @@ -83,7 +85,7 @@ class MatrixFederationHttpClient(object): query_bytes=b"", retry_on_dns_fail=True): """ Creates and sends a request to the given url """ - headers_dict[b"User-Agent"] = [b"Synapse"] + headers_dict[b"User-Agent"] = [AGENT_NAME] headers_dict[b"Host"] = [destination] url_bytes = urlparse.urlunparse( diff --git a/synapse/http/server.py b/synapse/http/server.py index f33859cf76..8015a22edf 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,14 +14,16 @@ # limitations under the License. -from syutil.jsonutil import ( - encode_canonical_json, encode_pretty_printed_json -) +from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException ) from synapse.util.logcontext import LoggingContext +from syutil.jsonutil import ( + encode_canonical_json, encode_pretty_printed_json +) + from twisted.internet import defer, reactor from twisted.web import server, resource from twisted.web.server import NOT_DONE_YET @@ -230,6 +232,8 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") + request.setHeader(b"Server", AGENT_NAME) + request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) if send_cors: request.setHeader("Access-Control-Allow-Origin", "*") diff --git a/synapse/http/server_key_resource.py b/synapse/http/server_key_resource.py index b30ecead27..4fc491dc82 100644 --- a/synapse/http/server_key_resource.py +++ b/synapse/http/server_key_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v0/content_repository.py b/synapse/media/v0/content_repository.py index ce5d3d153e..79ae0e3d74 100644 --- a/synapse/media/v0/content_repository.py +++ b/synapse/media/v0/content_repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v1/__init__.py b/synapse/media/v1/__init__.py index e69de29bb2..d6c6690577 100644 --- a/synapse/media/v1/__init__.py +++ b/synapse/media/v1/__init__.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import PIL.Image + +# check for JPEG support. +try: + PIL.Image._getdecoder("rgb", "jpeg", None) +except IOError as e: + if str(e).startswith("decoder jpeg not available"): + raise Exception( + "FATAL: jpeg codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" + " pip install pillow --user'" + ) +except Exception: + # any other exception is fine + pass + + +# check for PNG support. +try: + PIL.Image._getdecoder("rgb", "zip", None) +except IOError as e: + if str(e).startswith("decoder zip not available"): + raise Exception( + "FATAL: zip codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" + " pip install pillow --user'" + ) +except Exception: + # any other exception is fine + pass diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 2f5440ab64..688e7376ad 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -139,6 +139,7 @@ class BaseMediaResource(Resource): @download.addBoth def callback(media_info): del self.downloads[key] + return media_info return download @defer.inlineCallbacks @@ -201,7 +202,8 @@ class BaseMediaResource(Resource): defer.returnValue(media_info) @defer.inlineCallbacks - def _respond_with_file(self, request, media_type, file_path): + def _respond_with_file(self, request, media_type, file_path, + file_size=None): logger.debug("Responding with %r", file_path) if os.path.isfile(file_path): @@ -215,13 +217,20 @@ class BaseMediaResource(Resource): request.setHeader( b"Cache-Control", b"public,max-age=86400,s-maxage=86400" ) + if file_size is None: + stat = os.stat(file_path) + file_size = stat.st_size + + request.setHeader( + b"Content-Length", b"%d" % (file_size,) + ) with open(file_path, "rb") as f: yield FileSender().beginFileTransfer(f, request) request.finish() else: - self._respond_404() + self._respond_404(request) def _get_thumbnail_requirements(self, media_type): if media_type == "image/jpeg": diff --git a/synapse/media/v1/download_resource.py b/synapse/media/v1/download_resource.py index f3a6804e05..c585bb11f7 100644 --- a/synapse/media/v1/download_resource.py +++ b/synapse/media/v1/download_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -46,23 +46,29 @@ class DownloadResource(BaseMediaResource): def _respond_local_file(self, request, media_id): media_info = yield self.store.get_local_media(media_id) if not media_info: - self._respond_404() + self._respond_404(request) return media_type = media_info["media_type"] + media_length = media_info["media_length"] file_path = self.filepaths.local_media_filepath(media_id) - yield self._respond_with_file(request, media_type, file_path) + yield self._respond_with_file( + request, media_type, file_path, media_length + ) @defer.inlineCallbacks def _respond_remote_file(self, request, server_name, media_id): media_info = yield self._get_remote_media(server_name, media_id) media_type = media_info["media_type"] + media_length = media_info["media_length"] filesystem_id = media_info["filesystem_id"] file_path = self.filepaths.remote_media_filepath( server_name, filesystem_id ) - yield self._respond_with_file(request, media_type, file_path) + yield self._respond_with_file( + request, media_type, file_path, media_length + ) diff --git a/synapse/media/v1/filepath.py b/synapse/media/v1/filepath.py index 0078bc3d40..ed9a58e9d9 100644 --- a/synapse/media/v1/filepath.py +++ b/synapse/media/v1/filepath.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index 2070ec3c7f..461cc001f1 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/media/v1/thumbnail_resource.py b/synapse/media/v1/thumbnail_resource.py index e19620d456..84f5e3463c 100644 --- a/synapse/media/v1/thumbnail_resource.py +++ b/synapse/media/v1/thumbnail_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -100,11 +100,12 @@ class ThumbnailResource(BaseMediaResource): t_type = thumbnail_info["thumbnail_type"] t_method = thumbnail_info["thumbnail_method"] file_id = thumbnail_info["filesystem_id"] + t_length = thumbnail_info["thumbnail_length"] file_path = self.filepaths.remote_media_thumbnail( server_name, file_id, t_width, t_height, t_type, t_method, ) - yield self._respond_with_file(request, t_type, file_path) + yield self._respond_with_file(request, t_type, file_path, t_length) else: yield self._respond_default_thumbnail( request, media_info, width, height, method, m_type, @@ -139,11 +140,12 @@ class ThumbnailResource(BaseMediaResource): t_height = thumbnail_info["thumbnail_height"] t_type = thumbnail_info["thumbnail_type"] t_method = thumbnail_info["thumbnail_method"] + t_length = thumbnail_info["thumbnail_length"] file_path = self.filepaths.default_thumbnail( top_level_type, sub_type, t_width, t_height, t_type, t_method, ) - yield self.respond_with_file(request, t_type, file_path) + yield self.respond_with_file(request, t_type, file_path, t_length) def _select_thumbnail(self, desired_width, desired_height, desired_method, desired_type, thumbnail_infos): @@ -165,18 +167,27 @@ class ThumbnailResource(BaseMediaResource): aspect_quality, size_quality, type_quality, length_quality, info )) - return min(info_list)[-1] + if info_list: + return min(info_list)[-1] else: info_list = [] + info_list2 = [] for info in thumbnail_infos: t_w = info["thumbnail_width"] t_h = info["thumbnail_height"] t_method = info["thumbnail_method"] + size_quality = abs((d_w - t_w) * (d_h - t_h)) + type_quality = desired_type != info["thumbnail_type"] + length_quality = info["thumbnail_length"] if t_method == "scale" and (t_w >= d_w or t_h >= d_h): - size_quality = abs((d_w - t_w) * (d_h - t_h)) - type_quality = desired_type != info["thumbnail_type"] - length_quality = info["thumbnail_length"] info_list.append(( size_quality, type_quality, length_quality, info )) - return min(info_list)[-1] + elif t_method == "scale": + info_list2.append(( + size_quality, type_quality, length_quality, info + )) + if info_list: + return min(info_list)[-1] + else: + return min(info_list2)[-1] diff --git a/synapse/media/v1/thumbnailer.py b/synapse/media/v1/thumbnailer.py index 774ae4538f..28404f2b7b 100644 --- a/synapse/media/v1/thumbnailer.py +++ b/synapse/media/v1/thumbnailer.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ class Thumbnailer(object): def scale(self, output_path, width, height, output_type): """Rescales the image to the given dimensions""" - scaled = self.image.resize((width, height), Image.BILINEAR) + scaled = self.image.resize((width, height), Image.ANTIALIAS) return self.save_image(scaled, output_type, output_path) def crop(self, output_path, width, height, output_type): @@ -65,7 +65,7 @@ class Thumbnailer(object): if width * self.height > height * self.width: scaled_height = (width * self.height) // self.width scaled_image = self.image.resize( - (width, scaled_height), Image.BILINEAR + (width, scaled_height), Image.ANTIALIAS ) crop_top = (scaled_height - height) // 2 crop_bottom = height + crop_top @@ -73,7 +73,7 @@ class Thumbnailer(object): else: scaled_width = (height * self.width) // self.height scaled_image = self.image.resize( - (scaled_width, height), Image.BILINEAR + (scaled_width, height), Image.ANTIALIAS ) crop_left = (scaled_width - width) // 2 crop_right = width + crop_left @@ -82,7 +82,7 @@ class Thumbnailer(object): def save_image(self, output_image, output_type, output_path): output_bytes_io = BytesIO() - output_image.save(output_bytes_io, self.FORMATS[output_type]) + output_image.save(output_bytes_io, self.FORMATS[output_type], quality=70) output_bytes = output_bytes_io.getvalue() with open(output_path, "wb") as output_file: output_file.write(output_bytes) diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index 5645b0df46..b1718a630b 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/notifier.py b/synapse/notifier.py index 383230caf1..b9d52d0c4c 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py new file mode 100644 index 0000000000..b1fae991e0 --- /dev/null +++ b/synapse/python_dependencies.py @@ -0,0 +1,80 @@ +import logging +from distutils.version import LooseVersion + +logger = logging.getLogger(__name__) + +REQUIREMENTS = { + "syutil==0.0.2": ["syutil"], + "matrix_angular_sdk==0.6.0": ["syweb==0.6.0"], + "Twisted>=14.0.0": ["twisted>=14.0.0"], + "service_identity>=1.0.0": ["service_identity>=1.0.0"], + "pyopenssl>=0.14": ["OpenSSL>=0.14"], + "pyyaml": ["yaml"], + "pyasn1": ["pyasn1"], + "pynacl": ["nacl"], + "daemonize": ["daemonize"], + "py-bcrypt": ["bcrypt"], + "frozendict>=0.4": ["frozendict"], + "pillow": ["PIL"], +} + + +class MissingRequirementError(Exception): + pass + + +def check_requirements(): + """Checks that all the modules needed by synapse have been correctly + installed and are at the correct version""" + for dependency, module_requirements in REQUIREMENTS.items(): + for module_requirement in module_requirements: + if ">=" in module_requirement: + module_name, required_version = module_requirement.split(">=") + version_test = ">=" + elif "==" in module_requirement: + module_name, required_version = module_requirement.split("==") + version_test = "==" + else: + module_name = module_requirement + version_test = None + + try: + module = __import__(module_name) + except ImportError: + logging.exception( + "Can't import %r which is part of %r", + module_name, dependency + ) + raise MissingRequirementError( + "Can't import %r which is part of %r" + % (module_name, dependency) + ) + version = getattr(module, "__version__", None) + file_path = getattr(module, "__file__", None) + logger.info( + "Using %r version %r from %r to satisfy %r", + module_name, version, file_path, dependency + ) + + if version_test == ">=": + if version is None: + raise MissingRequirementError( + "Version of %r isn't set as __version__ of module %r" + % (dependency, module_name) + ) + if LooseVersion(version) < LooseVersion(required_version): + raise MissingRequirementError( + "Version of %r in %r is too old. %r < %r" + % (dependency, file_path, version, required_version) + ) + elif version_test == "==": + if version is None: + raise MissingRequirementError( + "Version of %r isn't set as __version__ of module %r" + % (dependency, module_name) + ) + if LooseVersion(version) != LooseVersion(required_version): + raise MissingRequirementError( + "Unexpected version of %r in %r. %r != %r" + % (dependency, file_path, version, required_version) + ) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index c29896bde9..59521d0c77 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/admin.py b/synapse/rest/admin.py index d74c551512..0aa83514c8 100644 --- a/synapse/rest/admin.py +++ b/synapse/rest/admin.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/base.py b/synapse/rest/base.py index 06eda2587c..c583945527 100644 --- a/synapse/rest/base.py +++ b/synapse/rest/base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py index 868fa7abed..7ff44fdd9e 100644 --- a/synapse/rest/directory.py +++ b/synapse/rest/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/events.py b/synapse/rest/events.py index 3c1b041bfe..bedcb2bcc6 100644 --- a/synapse/rest/events.py +++ b/synapse/rest/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -44,8 +44,11 @@ class EventStreamRestServlet(RestServlet): except ValueError: raise SynapseError(400, "timeout must be in milliseconds.") + as_client_event = "raw" not in request.args + chunk = yield handler.get_stream( - auth_user.to_string(), pagin_config, timeout=timeout + auth_user.to_string(), pagin_config, timeout=timeout, + as_client_event=as_client_event ) except: logger.exception("Event stream failed") diff --git a/synapse/rest/initial_sync.py b/synapse/rest/initial_sync.py index a1cb442256..b13d56b286 100644 --- a/synapse/rest/initial_sync.py +++ b/synapse/rest/initial_sync.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,12 +27,15 @@ class InitialSyncRestServlet(RestServlet): def on_GET(self, request): user = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args + as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) handler = self.handlers.message_handler content = yield handler.snapshot_all_rooms( user_id=user.to_string(), pagin_config=pagination_config, - feedback=with_feedback) + feedback=with_feedback, + as_client_event=as_client_event + ) defer.returnValue((200, content)) diff --git a/synapse/rest/login.py b/synapse/rest/login.py index 875da076af..6b8deff67b 100644 --- a/synapse/rest/login.py +++ b/synapse/rest/login.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/presence.py b/synapse/rest/presence.py index 9b42131628..ca4d2d21f0 100644 --- a/synapse/rest/presence.py +++ b/synapse/rest/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/profile.py b/synapse/rest/profile.py index 351863ab5d..dc6eb424b0 100644 --- a/synapse/rest/profile.py +++ b/synapse/rest/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 4f0f5a7531..e3b26902d9 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 005a9f6f44..caafa959e6 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -314,12 +314,15 @@ class RoomMessageListRestServlet(RestServlet): request, default_limit=10, ) with_feedback = "feedback" in request.args + as_client_event = "raw" not in request.args handler = self.handlers.message_handler msgs = yield handler.get_messages( room_id=room_id, user_id=user.to_string(), pagin_config=pagination_config, - feedback=with_feedback) + feedback=with_feedback, + as_client_event=as_client_event + ) defer.returnValue((200, msgs)) diff --git a/synapse/rest/transactions.py b/synapse/rest/transactions.py index 31377bd41d..d933fea18a 100644 --- a/synapse/rest/transactions.py +++ b/synapse/rest/transactions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/voip.py b/synapse/rest/voip.py index 432c2475f8..011c35e69b 100644 --- a/synapse/rest/voip.py +++ b/synapse/rest/voip.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/server.py b/synapse/server.py index b7d6811449..32d8a36db4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -151,8 +151,8 @@ class BaseHomeServer(object): object.""" return EventID.from_string(s) - def serialize_event(self, e): - return serialize_event(self, e) + def serialize_event(self, e, as_client_event=True): + return serialize_event(self, e, as_client_event) def get_ip_from_request(self, request): # May be an X-Forwarding-For header depending on config diff --git a/synapse/state.py b/synapse/state.py index 15397e2f38..8144fa02b4 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index ac3bf5cee5..fa7ad0eea8 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -68,7 +68,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 10 +SCHEMA_VERSION = 11 class _RollbackButIsFineException(Exception): @@ -146,9 +146,7 @@ class DataStore(RoomMemberStore, RoomStore, elif event.type == EventTypes.Redaction: self._store_redaction(txn, event) - outlier = False - if hasattr(event.internal_metadata, "outlier"): - outlier = event.internal_metadata.outlier + outlier = event.internal_metadata.is_outlier() event_dict = { k: v diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index efb2664680..4f172d3967 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -64,7 +64,7 @@ class LoggingTransaction(object): # Don't let logging failures stop SQL from working pass - start = time.clock() * 1000 + start = time.time() * 1000 try: return self.txn.execute( sql, *args, **kwargs @@ -73,7 +73,7 @@ class LoggingTransaction(object): logger.exception("[SQL FAIL] {%s}", self.name) raise finally: - end = time.clock() * 1000 + end = time.time() * 1000 sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) @@ -93,7 +93,7 @@ class SQLBaseStore(object): def inner_func(txn, *args, **kwargs): with LoggingContext("runInteraction") as context: current_context.copy_to(context) - start = time.clock() * 1000 + start = time.time() * 1000 txn_id = SQLBaseStore._TXN_ID # We don't really need these to be unique, so lets stop it from @@ -109,7 +109,7 @@ class SQLBaseStore(object): logger.exception("[TXN FAIL] {%s}", name) raise finally: - end = time.clock() * 1000 + end = time.time() * 1000 transaction_logger.debug( "[TXN END] {%s} %f", name, end - start @@ -479,23 +479,31 @@ class SQLBaseStore(object): return self.runInteraction("_simple_max_id", func) - def _get_events(self, event_ids): + def _get_events(self, event_ids, check_redacted=True, + get_prev_content=False): return self.runInteraction( - "_get_events", self._get_events_txn, event_ids + "_get_events", self._get_events_txn, event_ids, + check_redacted=check_redacted, get_prev_content=get_prev_content, ) - def _get_events_txn(self, txn, event_ids): - events = [] - for e_id in event_ids: - ev = self._get_event_txn(txn, e_id) + def _get_events_txn(self, txn, event_ids, check_redacted=True, + get_prev_content=False): + if not event_ids: + return [] - if ev: - events.append(ev) + events = [ + self._get_event_txn( + txn, event_id, + check_redacted=check_redacted, + get_prev_content=get_prev_content + ) + for event_id in event_ids + ] - return events + return [e for e in events if e] def _get_event_txn(self, txn, event_id, check_redacted=True, - get_prev_content=True): + get_prev_content=False): sql = ( "SELECT internal_metadata, json, r.event_id FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " @@ -512,6 +520,14 @@ class SQLBaseStore(object): internal_metadata, js, redacted = res + return self._get_event_from_row_txn( + txn, internal_metadata, js, redacted, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + + def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, + check_redacted=True, get_prev_content=False): d = json.loads(js) internal_metadata = json.loads(internal_metadata) @@ -533,11 +549,13 @@ class SQLBaseStore(object): ev.unsigned["redacted_because"] = because if get_prev_content and "replaces_state" in ev.unsigned: - ev.unsigned["prev_content"] = self._get_event_txn( + prev = self._get_event_txn( txn, ev.unsigned["replaces_state"], get_prev_content=False, - ).get_dict()["content"] + ) + if prev: + ev.unsigned["prev_content"] = prev.get_dict()["content"] return ev diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 2be9c41374..68b7d59693 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index ced066f407..0cbcdd1b55 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,39 +32,33 @@ class EventFederationStore(SQLBaseStore): and backfilling from another server respectively. """ - def get_auth_chain(self, event_id): + def get_auth_chain(self, event_ids): return self.runInteraction( "get_auth_chain", self._get_auth_chain_txn, - event_id + event_ids ) - def _get_auth_chain_txn(self, txn, event_id): - results = self._get_auth_chain_ids_txn(txn, event_id) + def _get_auth_chain_txn(self, txn, event_ids): + results = self._get_auth_chain_ids_txn(txn, event_ids) - sql = "SELECT * FROM events WHERE event_id = ?" - rows = [] - for ev_id in results: - c = txn.execute(sql, (ev_id,)) - rows.extend(self.cursor_to_dict(c)) + return self._get_events_txn(txn, results) - return self._parse_events_txn(txn, rows) - - def get_auth_chain_ids(self, event_id): + def get_auth_chain_ids(self, event_ids): return self.runInteraction( "get_auth_chain_ids", self._get_auth_chain_ids_txn, - event_id + event_ids ) - def _get_auth_chain_ids_txn(self, txn, event_id): + def _get_auth_chain_ids_txn(self, txn, event_ids): results = set() base_sql = ( "SELECT auth_id FROM event_auth WHERE %s" ) - front = set([event_id]) + front = set(event_ids) while front: sql = base_sql % ( " OR ".join(["event_id=?"] * len(front)), diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py index 21511577c5..fcf011b234 100644 --- a/synapse/storage/feedback.py +++ b/synapse/storage/feedback.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index fd705138e6..1f244019fc 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index 18c068d3d9..7101d2beec 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 71b2bb084d..1dcd34723b 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 7e1fdd9d88..153c7ad027 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 4d15005c9e..75dffa4db2 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 2378d65943..978b2c4a48 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -135,26 +135,26 @@ class RoomStore(SQLBaseStore): defer.returnValue(ret) def _store_room_topic_txn(self, txn, event): - if hasattr(event, "topic"): + if hasattr(event, "content") and "topic" in event.content: self._simple_insert_txn( txn, "topics", { "event_id": event.event_id, "room_id": event.room_id, - "topic": event.topic, + "topic": event.content["topic"], } ) def _store_room_name_txn(self, txn, event): - if hasattr(event, "name"): + if hasattr(event, "content") and "name" in event.content: self._simple_insert_txn( txn, "room_names", { "event_id": event.event_id, "room_id": event.room_id, - "name": event.name, + "name": event.content["name"], } ) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 05b275663e..e59e65529b 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ from twisted.internet import defer +from collections import namedtuple + from ._base import SQLBaseStore from synapse.api.constants import Membership @@ -24,6 +26,12 @@ import logging logger = logging.getLogger(__name__) +RoomsForUser = namedtuple( + "RoomsForUser", + ("room_id", "sender", "membership") +) + + class RoomMemberStore(SQLBaseStore): def _store_room_member_txn(self, txn, event): @@ -123,6 +131,19 @@ class RoomMemberStore(SQLBaseStore): else: return None + def get_users_in_room(self, room_id): + def f(txn): + sql = ( + "SELECT m.user_id FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE m.membership = ? AND m.room_id = ?" + ) + + txn.execute(sql, (Membership.JOIN, room_id)) + return [r[0] for r in txn.fetchall()] + return self.runInteraction("get_users_in_room", f) + def get_room_members(self, room_id, membership=None): """Retrieve the current room member list for a room. @@ -150,19 +171,37 @@ class RoomMemberStore(SQLBaseStore): membership_list (list): A list of synapse.api.constants.Membership values which the user must be in. Returns: - A list of RoomMemberEvent objects + A list of dictionary objects, with room_id, membership and sender + defined. """ if not membership_list: return defer.succeed(None) - args = [user_id] - args.extend(membership_list) - where_clause = "user_id = ? AND (%s)" % ( " OR ".join(["membership = ?" for _ in membership_list]), ) - return self._get_members_query(where_clause, args) + args = [user_id] + args.extend(membership_list) + + def f(txn): + sql = ( + "SELECT m.room_id, m.sender, m.membership" + " FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE %s" + ) % (where_clause,) + + txn.execute(sql, args) + return [ + RoomsForUser(**r) for r in self.cursor_to_dict(txn) + ] + + return self.runInteraction( + "get_rooms_for_user_where_membership_is", + f + ) def get_joined_hosts_for_room(self, room_id): return self._simple_select_onecol( @@ -183,20 +222,14 @@ class RoomMemberStore(SQLBaseStore): ) def _get_members_query_txn(self, txn, where_clause, where_values): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "SELECT e.* FROM events as e " "INNER JOIN room_memberships as m " "ON e.event_id = m.event_id " "INNER JOIN current_state_events as c " "ON m.event_id = c.event_id " "WHERE %(where)s " ) % { - "redacted": del_sql, "where": where_clause, } @@ -206,26 +239,28 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results - @defer.inlineCallbacks def user_rooms_intersect(self, user_id_list): """ Checks whether all the users whose IDs are given in a list share a room. """ - user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) - sql = ( - "SELECT m.room_id FROM room_memberships as m " - "INNER JOIN current_state_events as c " - "ON m.event_id = c.event_id " - "WHERE m.membership = 'join' " - "AND (%(clause)s) " - # TODO(paul): We've got duplicate rows in the database somewhere - # so we have to DISTINCT m.user_id here - "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" - ) % {"clause": user_list_clause} + def interaction(txn): + user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) + sql = ( + "SELECT m.room_id FROM room_memberships as m " + "INNER JOIN current_state_events as c " + "ON m.event_id = c.event_id " + "WHERE m.membership = 'join' " + "AND (%(clause)s) " + # TODO(paul): We've got duplicate rows in the database somewhere + # so we have to DISTINCT m.user_id here + "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" + ) % {"clause": user_list_clause} + + args = list(user_id_list) + args.append(len(user_id_list)) - args = list(user_id_list) - args.append(len(user_id_list)) + txn.execute(sql, args) - rows = yield self._execute(None, sql, *args) + return len(txn.fetchall()) > 0 - defer.returnValue(len(rows) > 0) + return self.runInteraction("user_rooms_intersect", interaction) diff --git a/synapse/storage/schema/delta/v11.sql b/synapse/storage/schema/delta/v11.sql new file mode 100644 index 0000000000..313592221b --- /dev/null +++ b/synapse/storage/schema/delta/v11.sql @@ -0,0 +1,16 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); \ No newline at end of file diff --git a/synapse/storage/schema/delta/v2.sql b/synapse/storage/schema/delta/v2.sql index 73b140465e..f740f6dd5d 100644 --- a/synapse/storage/schema/delta/v2.sql +++ b/synapse/storage/schema/delta/v2.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v3.sql b/synapse/storage/schema/delta/v3.sql index cade295989..c67e38ff52 100644 --- a/synapse/storage/schema/delta/v3.sql +++ b/synapse/storage/schema/delta/v3.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v4.sql b/synapse/storage/schema/delta/v4.sql index 25d2ead450..d3807b7686 100644 --- a/synapse/storage/schema/delta/v4.sql +++ b/synapse/storage/schema/delta/v4.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS redactions ( event_id TEXT NOT NULL, redacts TEXT NOT NULL, diff --git a/synapse/storage/schema/delta/v5.sql b/synapse/storage/schema/delta/v5.sql index af9df11aa9..0874a15431 100644 --- a/synapse/storage/schema/delta/v5.sql +++ b/synapse/storage/schema/delta/v5.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS user_ips ( user TEXT NOT NULL, diff --git a/synapse/storage/schema/delta/v6.sql b/synapse/storage/schema/delta/v6.sql index 9bf2068d84..a9e0a4fe0d 100644 --- a/synapse/storage/schema/delta/v6.sql +++ b/synapse/storage/schema/delta/v6.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v8.sql b/synapse/storage/schema/delta/v8.sql index daf6646ed5..1e9f8b18cb 100644 --- a/synapse/storage/schema/delta/v8.sql +++ b/synapse/storage/schema/delta/v8.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql index 0af29733a0..455d51a70c 100644 --- a/synapse/storage/schema/delta/v9.sql +++ b/synapse/storage/schema/delta/v9.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql index be1c72a775..1e766d6db2 100644 --- a/synapse/storage/schema/event_edges.sql +++ b/synapse/storage/schema/event_edges.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id TEXT NOT NULL, diff --git a/synapse/storage/schema/event_signatures.sql b/synapse/storage/schema/event_signatures.sql index b6b56b47a2..c28c39c48a 100644 --- a/synapse/storage/schema/event_signatures.sql +++ b/synapse/storage/schema/event_signatures.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 253f9f779b..dd00c1cd2f 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/keys.sql b/synapse/storage/schema/keys.sql index 9bf2068d84..a9e0a4fe0d 100644 --- a/synapse/storage/schema/keys.sql +++ b/synapse/storage/schema/keys.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/media_repository.sql b/synapse/storage/schema/media_repository.sql index b785fa0208..afdf48cbfb 100644 --- a/synapse/storage/schema/media_repository.sql +++ b/synapse/storage/schema/media_repository.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/presence.sql b/synapse/storage/schema/presence.sql index 595b3b5a69..f9f8db9697 100644 --- a/synapse/storage/schema/presence.sql +++ b/synapse/storage/schema/presence.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/profiles.sql b/synapse/storage/schema/profiles.sql index 58209f1af0..f06a528b4d 100644 --- a/synapse/storage/schema/profiles.sql +++ b/synapse/storage/schema/profiles.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/redactions.sql b/synapse/storage/schema/redactions.sql index 4c2829d05d..5011d95db8 100644 --- a/synapse/storage/schema/redactions.sql +++ b/synapse/storage/schema/redactions.sql @@ -1,3 +1,17 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS redactions ( event_id TEXT NOT NULL, redacts TEXT NOT NULL, diff --git a/synapse/storage/schema/room_aliases.sql b/synapse/storage/schema/room_aliases.sql index 9191016814..0d2df01603 100644 --- a/synapse/storage/schema/room_aliases.sql +++ b/synapse/storage/schema/room_aliases.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/state.sql index 2c48d6daca..1fe8f1e430 100644 --- a/synapse/storage/schema/state.sql +++ b/synapse/storage/schema/state.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index de461bfa15..2d30f99b06 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,7 @@ CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destinatio CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( destination ); +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); diff --git a/synapse/storage/schema/users.sql b/synapse/storage/schema/users.sql index 8244f733bd..08ccfdac0a 100644 --- a/synapse/storage/schema/users.sql +++ b/synapse/storage/schema/users.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 OpenMarket Ltd +/* Copyright 2014, 2015 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 3a705119fd..d0d53770f2 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/state.py b/synapse/storage/state.py index afe3e5edea..5327517704 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,10 @@ from ._base import SQLBaseStore +import logging + +logger = logging.getLogger(__name__) + class StateStore(SQLBaseStore): """ Keeps track of the state at a given event. @@ -62,14 +66,8 @@ class StateStore(SQLBaseStore): keyvalues={"state_group": group}, retcol="event_id", ) - state = [] - for state_id in state_ids: - s = self._get_events_txn( - txn, - [state_id], - ) - if s: - state.extend(s) + + state = self._get_events_txn(txn, state_ids) res[group] = state diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3405cb365e..744c821dfe 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -137,7 +137,6 @@ class StreamStore(SQLBaseStore): with_feedback=with_feedback, ) - @defer.inlineCallbacks @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, limit=0, with_feedback=False): @@ -157,11 +156,6 @@ class StreamStore(SQLBaseStore): "WHERE m.user_id = ? " ) - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - if limit: limit = max(limit, MAX_STREAM_SIZE) else: @@ -172,38 +166,42 @@ class StreamStore(SQLBaseStore): to_id = _parse_stream_token(to_key) if from_key == to_key: - defer.returnValue(([], to_key)) - return + return defer.succeed(([], to_key)) sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events AS e WHERE " + "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE " "(e.outlier = 0 AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { - "redacted": del_sql, "current": current_room_membership_sql, "invites": membership_sql, "limit": limit } - rows = yield self._execute_and_decode( - sql, - user_id, user_id, from_id, to_id - ) + def f(txn): + txn.execute(sql, (user_id, user_id, from_id, to_id,)) - ret = yield self._parse_events(rows) + rows = self.cursor_to_dict(txn) - if rows: - key = "s%d" % max([r["stream_ordering"] for r in rows]) - else: - # Assume we didn't get anything because there was nothing to get. - key = to_key + ret = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) + + if rows: + key = "s%d" % max([r["stream_ordering"] for r in rows]) + else: + # Assume we didn't get anything because there was nothing to + # get. + key = to_key + + return ret, key - defer.returnValue((ret, key)) + return self.runInteraction("get_room_events_stream", f) - @defer.inlineCallbacks @log_function def paginate_room_events(self, room_id, from_key, to_key=None, direction='b', limit=-1, @@ -221,7 +219,9 @@ class StreamStore(SQLBaseStore): bounds = _get_token_bound(from_key, from_comp) if to_key: - bounds = "%s AND %s" % (bounds, _get_token_bound(to_key, to_comp)) + bounds = "%s AND %s" % ( + bounds, _get_token_bound(to_key, to_comp) + ) if int(limit) > 0: args.append(int(limit)) @@ -229,87 +229,82 @@ class StreamStore(SQLBaseStore): else: limit_str = "" - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = events.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events" + "SELECT * FROM events" " WHERE outlier = 0 AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," " stream_ordering %(order)s %(limit)s" ) % { - "redacted": del_sql, "bounds": bounds, "order": order, "limit": limit_str } - rows = yield self._execute_and_decode( - sql, - *args - ) - - if rows: - topo = rows[-1]["topological_ordering"] - toke = rows[-1]["stream_ordering"] - if direction == 'b': - topo -= 1 - toke -= 1 - next_token = "t%s-%s" % (topo, toke) - else: - # TODO (erikj): We should work out what to do here instead. - next_token = to_key if to_key else from_key + def f(txn): + txn.execute(sql, args) + + rows = self.cursor_to_dict(txn) + + if rows: + topo = rows[-1]["topological_ordering"] + toke = rows[-1]["stream_ordering"] + if direction == 'b': + topo -= 1 + toke -= 1 + next_token = "t%s-%s" % (topo, toke) + else: + # TODO (erikj): We should work out what to do here instead. + next_token = to_key if to_key else from_key + + events = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) - events = yield self._parse_events(rows) + return events, next_token, - defer.returnValue( - ( - events, - next_token - ) - ) + return self.runInteraction("paginate_room_events", f) - @defer.inlineCallbacks def get_recent_events_for_room(self, room_id, limit, end_token, with_feedback=False): # TODO (erikj): Handle compressed feedback - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = events.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events " + "SELECT stream_ordering, topological_ordering, event_id FROM events " "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 " "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " - ) % { - "redacted": del_sql, - } - - rows = yield self._execute_and_decode( - sql, - room_id, end_token, limit ) - rows.reverse() # As we selected with reverse ordering + def f(txn): + txn.execute(sql, (room_id, end_token, limit,)) - if rows: - topo = rows[0]["topological_ordering"] - toke = rows[0]["stream_ordering"] - start_token = "t%s-%s" % (topo, toke) + rows = self.cursor_to_dict(txn) - token = (start_token, end_token) - else: - token = (end_token, end_token) + rows.reverse() # As we selected with reverse ordering - events = yield self._parse_events(rows) + if rows: + # XXX: Always subtract 1 since the start token always goes + # backwards (parity with paginate_room_events). It isn't + # obvious that this is correct; we should clarify the algorithm + # used here. + topo = rows[0]["topological_ordering"] - 1 + toke = rows[0]["stream_ordering"] - 1 + start_token = "t%s-%s" % (topo, toke) + + token = (start_token, end_token) + else: + token = (end_token, end_token) + + events = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) - ret = (events, token) + return events, token - defer.returnValue(ret) + return self.runInteraction("get_recent_events_for_room", f) def get_room_events_max_id(self): return self.runInteraction( diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 423cc3f02a..e06ef35690 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/__init__.py b/synapse/streams/__init__.py index f9811bfa04..c488b10d3c 100644 --- a/synapse/streams/__init__.py +++ b/synapse/streams/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/config.py b/synapse/streams/config.py index 2114c940e7..2ec7c5403b 100644 --- a/synapse/streams/config.py +++ b/synapse/streams/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/events.py b/synapse/streams/events.py index fb698d2d71..5c8e54b78b 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/types.py b/synapse/types.py index 7c533193e1..faac729ff2 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 7ec5033ceb..4e837a918e 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/async.py b/synapse/util/async.py index 7dd3ec3a72..c4fe5d522f 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 701ccdb781..9d9c350397 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -115,10 +115,10 @@ class Signal(object): failure.value, failure.getTracebackObject())) if not self.suppress_failures: - raise failure + failure.raiseException() deferreds.append(d.addErrback(eb)) - - result = yield defer.DeferredList( - deferreds, fireOnOneErrback=not self.suppress_failures - ) - defer.returnValue(result) + results = [] + for deferred in deferreds: + result = yield deferred + results.append(result) + defer.returnValue(results) diff --git a/synapse/util/emailutils.py b/synapse/util/emailutils.py index 7038cab6c2..7f9a77bf44 100644 --- a/synapse/util/emailutils.py +++ b/synapse/util/emailutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index fcfb38b3b4..a13a2015e4 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,10 +17,11 @@ from frozendict import frozendict def freeze(o): - if isinstance(o, dict) or isinstance(o, frozendict): + t = type(o) + if t is dict: return frozendict({k: freeze(v) for k, v in o.items()}) - if isinstance(o, basestring): + if t is str or t is unicode: return o try: diff --git a/synapse/util/jsonobject.py b/synapse/util/jsonobject.py index e79b68f661..0765f7d217 100644 --- a/synapse/util/jsonobject.py +++ b/synapse/util/jsonobject.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/lockutils.py b/synapse/util/lockutils.py index 3a84c09db4..33edc5c20e 100644 --- a/synapse/util/lockutils.py +++ b/synapse/util/lockutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 7d85018d97..da7872e95d 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -1,3 +1,17 @@ +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import threading import logging diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index 903a6cf1b3..fd9ac4d4d4 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,14 +19,37 @@ from functools import wraps import logging import inspect +import time + + +_TIME_FUNC_ID = 0 + + +def _log_debug_as_f(f, msg, msg_args): + name = f.__module__ + logger = logging.getLogger(name) + + if logger.isEnabledFor(logging.DEBUG): + lineno = f.func_code.co_firstlineno + pathname = f.func_code.co_filename + + record = logging.LogRecord( + name=name, + level=logging.DEBUG, + pathname=pathname, + lineno=lineno, + msg=msg, + args=msg_args, + exc_info=None + ) + + logger.handle(record) def log_function(f): """ Function decorator that logs every call to that function. """ func_name = f.__name__ - lineno = f.func_code.co_firstlineno - pathname = f.func_code.co_filename @wraps(f) def wrapped(*args, **kwargs): @@ -52,24 +75,50 @@ def log_function(f): "args": ", ".join(func_args) } - record = logging.LogRecord( - name=name, - level=level, - pathname=pathname, - lineno=lineno, - msg="Invoked '%(func_name)s' with args: %(args)s", - args=msg_args, - exc_info=None + _log_debug_as_f( + f, + "Invoked '%(func_name)s' with args: %(args)s", + msg_args ) - logger.handle(record) - return f(*args, **kwargs) wrapped.__name__ = func_name return wrapped +def time_function(f): + func_name = f.__name__ + + @wraps(f) + def wrapped(*args, **kwargs): + global _TIME_FUNC_ID + id = _TIME_FUNC_ID + _TIME_FUNC_ID += 1 + + start = time.clock() * 1000 + + try: + _log_debug_as_f( + f, + "[FUNC START] {%s-%d}", + (func_name, id), + ) + + r = f(*args, **kwargs) + finally: + end = time.clock() * 1000 + _log_debug_as_f( + f, + "[FUNC END] {%s-%d} %f", + (func_name, id, end-start,), + ) + + return r + + return wrapped + + def trace_function(f): func_name = f.__name__ linenum = f.func_code.co_firstlineno diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index 8767e437dd..ea53a8085c 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2014, 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 79ac1ce10d..3e484cd303 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -52,6 +52,7 @@ class FederationTestCase(unittest.TestCase): "get_received_txn_response", "set_received_txn_response", "get_destination_retry_timings", + "get_auth_chain", ]) self.mock_persistence.get_received_txn_response.return_value = ( defer.succeed(None) @@ -59,6 +60,7 @@ class FederationTestCase(unittest.TestCase): self.mock_persistence.get_destination_retry_timings.return_value = ( defer.succeed(DestinationsTable.EntryType("", 0, 0)) ) + self.mock_persistence.get_auth_chain.return_value = [] self.mock_config = Mock() self.mock_config.signing_key = [MockKey()] self.clock = MockClock() diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 532ecf0f2c..0584e4c8b9 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -19,7 +19,7 @@ presence and profiles; namely, the displayname and avatar_url.""" from tests import unittest from twisted.internet import defer -from mock import Mock, call, ANY +from mock import Mock, call, ANY, NonCallableMock from ..utils import MockClock, MockKey @@ -75,8 +75,13 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): resource_for_federation=Mock(), http_client=None, replication_layer=MockReplication(), - config=self.mock_config, - ) + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) hs.handlers = PresenceAndProfileHandlers(hs) self.datastore = hs.get_datastore() diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 1660e7e928..25b172aa5e 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -17,7 +17,7 @@ from tests import unittest from twisted.internet import defer -from mock import Mock +from mock import Mock, NonCallableMock from synapse.api.errors import AuthError from synapse.server import HomeServer @@ -59,7 +59,14 @@ class ProfileTestCase(unittest.TestCase): resource_for_federation=Mock(), replication_layer=self.mock_federation, config=self.mock_config, + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]) ) + + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + hs.handlers = ProfileHandlers(hs) self.store = hs.get_datastore() diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 0d4b368a43..6a498b23a4 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -352,3 +352,29 @@ class TypingNotificationsTestCase(unittest.TestCase): }}, ] ) + + # SYN-230 - see if we can still set after timeout + + yield self.handler.started_typing( + target_user=self.u_apple, + auth_user=self.u_apple, + room_id=self.room_id, + timeout=10000, + ) + + self.on_new_user_event.assert_has_calls([ + call(rooms=[self.room_id]), + ]) + self.on_new_user_event.reset_mock() + + self.assertEquals(self.event_source.get_current_key(), 3) + self.assertEquals( + self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + [ + {"type": "m.typing", + "room_id": self.room_id, + "content": { + "user_ids": [self.u_apple.to_string()], + }}, + ] + ) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 84fd730afc..8e65ff9a1c 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -294,7 +294,7 @@ class RoomPermissionsTestCase(RestTestCase): # set [invite/join/left] of self, set [invite/join/left] of other, # expect all 403s for usr in [self.user_id, self.rmcreator_id]: - yield self.join(room=room, user=usr, expect_code=403) + yield self.join(room=room, user=usr, expect_code=404) yield self.leave(room=room, user=usr, expect_code=403) @defer.inlineCallbacks diff --git a/tests/rest/test_typing.py b/tests/rest/test_typing.py index c550294d59..18138af1b5 100644 --- a/tests/rest/test_typing.py +++ b/tests/rest/test_typing.py @@ -21,7 +21,7 @@ from twisted.internet import defer import synapse.rest.room from synapse.server import HomeServer -from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from ..utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -36,6 +36,8 @@ class RoomTypingTestCase(RestTestCase): @defer.inlineCallbacks def setUp(self): + self.clock = MockClock() + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id @@ -47,6 +49,7 @@ class RoomTypingTestCase(RestTestCase): hs = HomeServer( "red", + clock=self.clock, db_pool=db_pool, http_client=None, replication_layer=Mock(), @@ -77,6 +80,30 @@ class RoomTypingTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip + def get_room_members(room_id): + if room_id == self.room_id: + return defer.succeed([hs.parse_userid(self.user_id)]) + else: + return defer.succeed([]) + + @defer.inlineCallbacks + def fetch_room_distributions_into(room_id, localusers=None, + remotedomains=None, ignore_user=None): + + members = yield get_room_members(room_id) + for member in members: + if ignore_user is not None and member == ignore_user: + continue + + if hs.is_mine(member): + if localusers is not None: + localusers.add(member) + else: + if remotedomains is not None: + remotedomains.add(member.domain) + hs.get_handlers().room_member_handler.fetch_room_distributions_into = ( + fetch_room_distributions_into) + synapse.rest.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) @@ -113,3 +140,25 @@ class RoomTypingTestCase(RestTestCase): '{"typing": false}' ) self.assertEquals(200, code) + + @defer.inlineCallbacks + def test_typing_timeout(self): + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": true, "timeout": 30000}' + ) + self.assertEquals(200, code) + + self.assertEquals(self.event_source.get_current_key(), 1) + + self.clock.advance_time(31); + + self.assertEquals(self.event_source.get_current_key(), 2) + + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": true, "timeout": 30000}' + ) + self.assertEquals(200, code) + + self.assertEquals(self.event_source.get_current_key(), 3) diff --git a/tests/test_distributor.py b/tests/test_distributor.py index 39c5b8dff2..6a0095d850 100644 --- a/tests/test_distributor.py +++ b/tests/test_distributor.py @@ -13,12 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from tests import unittest +from . import unittest from twisted.internet import defer from mock import Mock, patch from synapse.util.distributor import Distributor +from synapse.util.async import run_on_reactor class DistributorTestCase(unittest.TestCase): @@ -26,6 +27,7 @@ class DistributorTestCase(unittest.TestCase): def setUp(self): self.dist = Distributor() + @defer.inlineCallbacks def test_signal_dispatch(self): self.dist.declare("alert") @@ -33,10 +35,11 @@ class DistributorTestCase(unittest.TestCase): self.dist.observe("alert", observer) d = self.dist.fire("alert", 1, 2, 3) - + yield d self.assertTrue(d.called) observer.assert_called_with(1, 2, 3) + @defer.inlineCallbacks def test_signal_dispatch_deferred(self): self.dist.declare("whine") @@ -50,8 +53,10 @@ class DistributorTestCase(unittest.TestCase): self.assertFalse(d_outer.called) d_inner.callback(None) + yield d_outer self.assertTrue(d_outer.called) + @defer.inlineCallbacks def test_signal_catch(self): self.dist.declare("alarm") @@ -65,6 +70,7 @@ class DistributorTestCase(unittest.TestCase): spec=["warning"] ) as mock_logger: d = self.dist.fire("alarm", "Go") + yield d self.assertTrue(d.called) observers[0].assert_called_once("Go") @@ -81,23 +87,28 @@ class DistributorTestCase(unittest.TestCase): self.dist.declare("whail") - observer = Mock() - observer.return_value = defer.fail( - Exception("Oopsie") - ) + class MyException(Exception): + pass + + @defer.inlineCallbacks + def observer(): + yield run_on_reactor() + raise MyException("Oopsie") self.dist.observe("whail", observer) d = self.dist.fire("whail") - yield self.assertFailure(d, Exception) + yield self.assertFailure(d, MyException) + self.dist.suppress_failures = True + @defer.inlineCallbacks def test_signal_prereg(self): observer = Mock() self.dist.observe("flare", observer) self.dist.declare("flare") - self.dist.fire("flare", 4, 5) + yield self.dist.fire("flare", 4, 5) observer.assert_called_with(4, 5) diff --git a/tests/unittest.py b/tests/unittest.py index a9c0e05541..fe26b7574f 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -69,6 +69,8 @@ class TestCase(unittest.TestCase): return ret logging.getLogger().setLevel(level) + # Don't set SQL logging + logging.getLogger("synapse.storage").setLevel(old_level) return orig() def assertObjectHasAttributes(self, attrs, obj): |