diff options
Diffstat (limited to 'synapse')
98 files changed, 1086 insertions, 377 deletions
diff --git a/synapse/__init__.py b/synapse/__init__.py index b45cf47b56..440e633966 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.2.0" +__version__ = "0.2.1" diff --git a/synapse/api/__init__.py b/synapse/api/__init__.py index 2216c0f1ca..9bff9ec169 100644 --- a/synapse/api/__init__.py +++ b/synapse/api/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 54ecbe5b3a..b4eda3df01 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 668ffa07ca..fcef062fc9 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 21ededc5ae..84afe4fa37 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ class Codes(object): UNKNOWN = "M_UNKNOWN" NOT_FOUND = "M_NOT_FOUND" UNKNOWN_TOKEN = "M_UNKNOWN_TOKEN" + LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" class CodeMessageException(Exception): @@ -38,11 +39,15 @@ class CodeMessageException(Exception): super(CodeMessageException, self).__init__("%d: %s" % (code, msg)) self.code = code self.msg = msg + self.response_code_message = None + + def error_dict(self): + return cs_error(self.msg) class SynapseError(CodeMessageException): """A base error which can be caught for all synapse events.""" - def __init__(self, code, msg, errcode=""): + def __init__(self, code, msg, errcode=Codes.UNKNOWN): """Constructs a synapse error. Args: @@ -53,6 +58,11 @@ class SynapseError(CodeMessageException): super(SynapseError, self).__init__(code, msg) self.errcode = errcode + def error_dict(self): + return cs_error( + self.msg, + self.errcode, + ) class RoomError(SynapseError): """An error raised when a room event fails.""" @@ -91,13 +101,26 @@ class StoreError(SynapseError): pass -def cs_exception(exception): - if isinstance(exception, SynapseError): +class LimitExceededError(SynapseError): + """A client has sent too many requests and is being throttled. + """ + def __init__(self, code=429, msg="Too Many Requests", retry_after_ms=None, + errcode=Codes.LIMIT_EXCEEDED): + super(LimitExceededError, self).__init__(code, msg, errcode) + self.retry_after_ms = retry_after_ms + self.response_code_message = "Too Many Requests" + + def error_dict(self): return cs_error( - exception.msg, - Codes.UNKNOWN if not exception.errcode else exception.errcode) - elif isinstance(exception, CodeMessageException): - return cs_error(exception.msg) + self.msg, + self.errcode, + retry_after_ms=self.retry_after_ms, + ) + + +def cs_exception(exception): + if isinstance(exception, CodeMessageException): + return exception.error_dict() else: logging.error("Unknown exception type: %s", type(exception)) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 9502f5df8f..f95468fc65 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 159728b2d2..a3b293e024 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index f6d3c59a9a..33f0f0cb99 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -103,8 +103,7 @@ class FeedbackEvent(SynapseEvent): def get_content_template(self): return { "type": u"string", - "target_event_id": u"string", - "msg_sender_id": u"string" + "target_event_id": u"string" } diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py new file mode 100644 index 0000000000..b25358090f --- /dev/null +++ b/synapse/api/ratelimiting.py @@ -0,0 +1,79 @@ +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + + +class Ratelimiter(object): + """ + Ratelimit message sending by user. + """ + + def __init__(self): + self.message_counts = collections.OrderedDict() + + def send_message(self, user_id, time_now_s, msg_rate_hz, burst_count): + """Can the user send a message? + Args: + user_id: The user sending a message. + time_now_s: The time now. + msg_rate_hz: The long term number of messages a user can send in a + second. + burst_count: How many messages the user can send before being + limited. + Returns: + A pair of a bool indicating if they can send a message now and a + time in seconds of when they can next send a message. + """ + self.prune_message_counts(time_now_s) + message_count, time_start, _ignored = self.message_counts.pop( + user_id, (0., time_now_s, None), + ) + time_delta = time_now_s - time_start + sent_count = message_count - time_delta * msg_rate_hz + if sent_count < 0: + allowed = True + time_start = time_now_s + message_count = 1. + elif sent_count > burst_count - 1.: + allowed = False + else: + allowed = True + message_count += 1 + + self.message_counts[user_id] = ( + message_count, time_start, msg_rate_hz + ) + + if msg_rate_hz > 0: + time_allowed = ( + time_start + (message_count - burst_count + 1) / msg_rate_hz + ) + if time_allowed < time_now_s: + time_allowed = time_now_s + else: + time_allowed = -1 + + return allowed, time_allowed + + def prune_message_counts(self, time_now_s): + for user_id in self.message_counts.keys(): + message_count, time_start, msg_rate_hz = ( + self.message_counts[user_id] + ) + time_delta = time_now_s - time_start + if message_count - time_delta * msg_rate_hz > 0: + break + else: + del self.message_counts[user_id] diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 3d0b5de965..6314f31f7a 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index 2216c0f1ca..9bff9ec169 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 606c9c650d..49cf928cc1 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,7 +23,8 @@ from twisted.enterprise import adbapi from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site -from synapse.http.server import JsonResource, RootRedirect, ContentRepoResource +from synapse.http.server import JsonResource, RootRedirect +from synapse.http.content_repository import ContentRepoResource from synapse.http.client import TwistedHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX @@ -74,7 +75,9 @@ class SynapseHomeServer(HomeServer): return File("webclient") # TODO configurable? def build_resource_for_content_repo(self): - return ContentRepoResource(self, self.upload_dir, self.auth) + return ContentRepoResource( + self, self.upload_dir, self.auth, self.content_addr + ) def build_db_pool(self): """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we @@ -90,20 +93,28 @@ class SynapseHomeServer(HomeServer): if row and row[0]: user_version = row[0] - if user_version < SCHEMA_VERSION: - # TODO(paul): add some kind of intelligent fixup here - raise ValueError("Cannot use this database as the " + - "schema version (%d) does not match (%d)" % - (user_version, SCHEMA_VERSION) + if user_version > SCHEMA_VERSION: + raise ValueError("Cannot use this database as it is too " + + "new for the server to understand" + ) + elif user_version < SCHEMA_VERSION: + logging.info("Upgrading database from version %d", + user_version ) + # Run every version since after the current version. + for v in range(user_version + 1, SCHEMA_VERSION + 1): + sql_script = read_schema("delta/v%d" % (v)) + c.executescript(sql_script) + + db_conn.commit() + else: for sql_loc in SCHEMAS: sql_script = read_schema(sql_loc) c.executescript(sql_script) - db_conn.commit() - + db_conn.commit() c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION) c.close() @@ -247,6 +258,8 @@ def setup(): upload_dir=os.path.abspath("uploads"), db_name=config.database_path, tls_context_factory=tls_context_factory, + config=config, + content_addr=config.content_addr, ) hs.register_servlets() diff --git a/synapse/config/__init__.py b/synapse/config/__init__.py index fe8a073cd3..f9811bfa04 100644 --- a/synapse/config/__init__.py +++ b/synapse/config/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 1913179c3a..35bcece2c0 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - -import ConfigParser as configparser import argparse import sys import os @@ -121,6 +119,9 @@ class Config(object): and value is not None): config[key] = value with open(config_args.config_path, "w") as config_file: + # TODO(paul) it would be lovely if we wrote out vim- and emacs- + # style mode markers into the file, to hint to people that + # this is a YAML file. yaml.dump(config, config_file, default_flow_style=False) sys.exit(0) diff --git a/synapse/config/database.py b/synapse/config/database.py index edf2361914..460445f15d 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 18072e3196..76e2cdeddd 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,8 +17,11 @@ from .tls import TlsConfig from .server import ServerConfig from .logger import LoggingConfig from .database import DatabaseConfig +from .ratelimiting import RatelimitConfig +from .repository import ContentRepositoryConfig -class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig): +class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, + RatelimitConfig, ContentRepositoryConfig): pass if __name__=='__main__': diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 8db6621ae8..56cd095433 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py new file mode 100644 index 0000000000..f126782b8d --- /dev/null +++ b/synapse/config/ratelimiting.py @@ -0,0 +1,35 @@ +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + +class RatelimitConfig(Config): + + def __init__(self, args): + super(RatelimitConfig, self).__init__(args) + self.rc_messages_per_second = args.rc_messages_per_second + self.rc_message_burst_count = args.rc_message_burst_count + + @classmethod + def add_arguments(cls, parser): + super(RatelimitConfig, cls).add_arguments(parser) + rc_group = parser.add_argument_group("ratelimiting") + rc_group.add_argument( + "--rc-messages-per-second", type=float, default=0.2, + help="number of messages a client can send per second" + ) + rc_group.add_argument( + "--rc-message-burst-count", type=float, default=10, + help="number of message a client can send before being throttled" + ) diff --git a/synapse/config/repository.py b/synapse/config/repository.py new file mode 100644 index 0000000000..407c8d6c24 --- /dev/null +++ b/synapse/config/repository.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 matrix.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config +import os + +class ContentRepositoryConfig(Config): + def __init__(self, args): + super(ContentRepositoryConfig, self).__init__(args) + self.max_upload_size = self.parse_size(args.max_upload_size) + + def parse_size(self, string): + sizes = {"K": 1024, "M": 1024 * 1024} + size = 1 + suffix = string[-1] + if suffix in sizes: + string = string[:-1] + size = sizes[suffix] + return int(string) * size + + @classmethod + def add_arguments(cls, parser): + super(ContentRepositoryConfig, cls).add_arguments(parser) + db_group = parser.add_argument_group("content_repository") + db_group.add_argument( + "--max-upload-size", default="1M" + ) diff --git a/synapse/config/server.py b/synapse/config/server.py index 36143e3c9c..516e4cf882 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,6 +32,14 @@ class ServerConfig(Config): self.webclient = True self.manhole = args.manhole + if not args.content_addr: + host = args.server_name + if ':' not in host: + host = "%s:%d" % (host, args.bind_port) + args.content_addr = "https://%s" % (host,) + + self.content_addr = args.content_addr + @classmethod def add_arguments(cls, parser): super(ServerConfig, cls).add_arguments(parser) @@ -50,13 +58,16 @@ class ServerConfig(Config): help="Local interface to listen on") server_group.add_argument("-D", "--daemonize", action='store_true', help="Daemonize the home server") - server_group.add_argument('--pid-file', default="hs.pid", + server_group.add_argument('--pid-file', default="homeserver.pid", help="When running as a daemon, the file to" " store the pid in") server_group.add_argument("--manhole", metavar="PORT", dest="manhole", type=int, help="Turn on the twisted telnet manhole" " service on the given port.") + server_group.add_argument("--content-addr", default=None, + help="The host and scheme to use for the " + "content repository") def read_signing_key(self, signing_key_path): signing_key_base64 = self.read_file(signing_key_path, "signing_key") @@ -77,3 +88,4 @@ class ServerConfig(Config): with open(args.signing_key_path, "w") as signing_key_file: key = nacl.signing.SigningKey.generate() signing_key_file.write(encode_base64(key.encode())) + diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 16f6f3aba6..72d5518a89 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/__init__.py b/synapse/crypto/__init__.py index 2216c0f1ca..9bff9ec169 100644 --- a/synapse/crypto/__init__.py +++ b/synapse/crypto/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 45958abbf5..f86bd19255 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -1,4 +1,18 @@ -from twisted.internet import reactor, ssl +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import ssl from OpenSSL import SSL from twisted.internet._sslverify import _OpenSSLECCurve, _defaultCurveName diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index e615866b68..c11df5c529 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/keyserver.py b/synapse/crypto/keyserver.py index 3d80a0e660..a23484dbae 100644 --- a/synapse/crypto/keyserver.py +++ b/synapse/crypto/keyserver.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/resource/__init__.py b/synapse/crypto/resource/__init__.py index 2216c0f1ca..9bff9ec169 100644 --- a/synapse/crypto/resource/__init__.py +++ b/synapse/crypto/resource/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/crypto/resource/key.py b/synapse/crypto/resource/key.py index 6aecd2b95f..48d14b9f4a 100644 --- a/synapse/crypto/resource/key.py +++ b/synapse/crypto/resource/key.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/__init__.py b/synapse/federation/__init__.py index b15e7cf941..1351b68fd6 100644 --- a/synapse/federation/__init__.py +++ b/synapse/federation/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index adc166c564..cef61108dd 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 4cf72b2e42..de36a80e41 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index cadf574b3b..e12510017f 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -628,7 +628,6 @@ class _TransactionQueue(object): for deferred in deferreds: deferred.errback(e) - yield deferred finally: # We want to be *very* sure we delete this after we stop processing diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 50c3df4a5d..6e62ae7c74 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/federation/units.py b/synapse/federation/units.py index b468f70546..9740431279 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index b2208b26c3..5308e2c8e1 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index b37c8be964..9989fe8670 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ # limitations under the License. from twisted.internet import defer +from synapse.api.errors import LimitExceededError class BaseHandler(object): @@ -25,8 +26,22 @@ class BaseHandler(object): self.room_lock = hs.get_room_lock_manager() self.state_handler = hs.get_state_handler() self.distributor = hs.get_distributor() + self.ratelimiter = hs.get_ratelimiter() + self.clock = hs.get_clock() self.hs = hs + def ratelimit(self, user_id): + time_now = self.clock.time() + allowed, time_allowed = self.ratelimiter.send_message( + user_id, time_now, + msg_rate_hz=self.hs.config.rc_messages_per_second, + burst_count=self.hs.config.rc_message_burst_count, + ) + if not allowed: + raise LimitExceededError( + retry_after_ms=int(1000*(time_allowed - time_now)), + ) + class BaseRoomHandler(BaseHandler): diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 7c89150d99..1b9e831fc0 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError +from synapse.http.client import HttpClient import logging @@ -36,7 +37,7 @@ class DirectoryHandler(BaseHandler): ) @defer.inlineCallbacks - def create_association(self, room_alias, room_id, servers): + def create_association(self, room_alias, room_id, servers=None): # TODO(erikj): Do auth. if not room_alias.is_mine: @@ -47,6 +48,12 @@ class DirectoryHandler(BaseHandler): # TODO(erikj): Check if there is a current association. + if not servers: + servers = yield self.store.get_joined_hosts_for_room(room_id) + + if not servers: + raise SynapseError(400, "Failed to get server list") + yield self.store.create_room_alias_association( room_alias, room_id, @@ -68,7 +75,10 @@ class DirectoryHandler(BaseHandler): result = yield self.federation.make_query( destination=room_alias.domain, query_type="directory", - args={"room_alias": room_alias.to_string()}, + args={ + "room_alias": room_alias.to_string(), + HttpClient.RETRY_DNS_LOOKUP_FAILURES: False + } ) if result and "room_id" in result and "servers" in result: @@ -79,6 +89,9 @@ class DirectoryHandler(BaseHandler): defer.returnValue({}) return + extra_servers = yield self.store.get_joined_hosts_for_room(room_id) + servers = list(set(extra_servers) | set(servers)) + defer.returnValue({ "room_id": room_id, "servers": servers, diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 980a169b25..fd24a11fb8 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -126,5 +126,7 @@ class EventHandler(BaseHandler): defer.returnValue(None) return - yield self.auth.check(event, raises=True) + if hasattr(event, "room_id"): + yield self.auth.check_joined_room(event.room_id, user.to_string()) + defer.returnValue(event) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index eac110419c..59cbf71d78 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,8 +21,9 @@ from synapse.api.events.room import InviteJoinEvent, RoomMemberEvent from synapse.api.constants import Membership from synapse.util.logutils import log_function from synapse.federation.pdu_codec import PduCodec +from synapse.api.errors import SynapseError -from twisted.internet import defer +from twisted.internet import defer, reactor import logging @@ -133,7 +134,7 @@ class FederationHandler(BaseHandler): yield self.hs.get_handlers().room_member_handler.change_membership( new_event, - do_auth=True + do_auth=False, ) else: @@ -231,7 +232,12 @@ class FederationHandler(BaseHandler): # TODO (erikj): Time out here. d = defer.Deferred() self.waiting_for_join_list.setdefault((joinee, room_id), []).append(d) - yield d + reactor.callLater(10, d.cancel) + + try: + yield d + except defer.CancelledError: + raise SynapseError(500, "Unable to join remote room") try: yield self.store.store_room( diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 0220fa0604..6ee7ce5a2d 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 4aeb2089f5..dad2bbd1a4 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -76,6 +76,8 @@ class MessageHandler(BaseRoomHandler): Raises: SynapseError if something went wrong. """ + + self.ratelimit(event.user_id) # TODO(paul): Why does 'event' not have a 'user' object? user = self.hs.parse_userid(event.user_id) assert user.is_mine, "User must be our own: %s" % (user,) @@ -140,7 +142,12 @@ class MessageHandler(BaseRoomHandler): SynapseError if something went wrong. """ - snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) + snapshot = yield self.store.snapshot_room( + event.room_id, + event.user_id, + state_type=event.type, + state_key=event.state_key, + ) yield self.auth.check(event, snapshot, raises=True) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 9bfceda88a..c79bb6ff76 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -155,19 +155,18 @@ class PresenceHandler(BaseHandler): if observer_user == observed_user: defer.returnValue(True) - allowed_by_subscription = yield self.store.is_presence_visible( + if (yield self.store.user_rooms_intersect( + [u.to_string() for u in observer_user, observed_user] + )): + defer.returnValue(True) + + if (yield self.store.is_presence_visible( observed_localpart=observed_user.localpart, observer_userid=observer_user.to_string(), - ) - - if allowed_by_subscription: + )): defer.returnValue(True) - share_room = yield self.store.do_users_share_a_room( - [observer_user, observed_user] - ) - - defer.returnValue(share_room) + defer.returnValue(False) @defer.inlineCallbacks def get_state(self, target_user, auth_user): @@ -181,7 +180,7 @@ class PresenceHandler(BaseHandler): state = yield self.store.get_presence_state(target_user.localpart) if "mtime" in state: del state["mtime"] - state["presence"] = state["state"] + state["presence"] = state.pop("state") if target_user in self._user_cachemap: state["last_active"] = ( @@ -208,21 +207,17 @@ class PresenceHandler(BaseHandler): raise SynapseError(400, "User is not hosted on this Home Server") if target_user != auth_user: - raise AuthError(400, "Cannot set another user's displayname") + raise AuthError(400, "Cannot set another user's presence") if "status_msg" not in state: state["status_msg"] = None for k in state.keys(): - if k not in ("presence", "state", "status_msg"): + if k not in ("presence", "status_msg"): raise SynapseError( 400, "Unexpected presence state key '%s'" % (k,) ) - # Handle legacy "state" key for now - if "state" in state: - state["presence"] = state.pop("state") - if state["presence"] not in self.STATE_LEVELS: raise SynapseError(400, "'%s' is not a valid presence state" % state["presence"] @@ -601,7 +596,7 @@ class PresenceHandler(BaseHandler): if state is None: state = yield self.store.get_presence_state(user.localpart) del state["mtime"] - state["presence"] = state["state"] + state["presence"] = state.pop("state") if user in self._user_cachemap: state["last_active"] = ( @@ -622,8 +617,6 @@ class PresenceHandler(BaseHandler): "user_id": user.to_string(), } user_state.update(**state) - if "state" in user_state and "presence" not in user_state: - user_state["presence"] = user_state["state"] yield self.federation.send_edu( destination=destination, @@ -655,21 +648,12 @@ class PresenceHandler(BaseHandler): state = dict(push) del state["user_id"] - if "presence" in state: - # all is OK - pass - elif "state" in state: - # Legacy handling - state["presence"] = state["state"] - else: + if "presence" not in state: logger.warning("Received a presence 'push' EDU from %s without" - + " either a 'presence' or 'state' key", origin + + " a 'presence' key", origin ) continue - if "state" in state: - del state["state"] - if "last_active_ago" in state: state["last_active"] = int( self.clock.time_msec() - state.pop("last_active_ago") @@ -773,15 +757,52 @@ class PresenceEventSource(object): self.hs = hs self.clock = hs.get_clock() + @defer.inlineCallbacks + def is_visible(self, observer_user, observed_user): + if observer_user == observed_user: + defer.returnValue(True) + + presence = self.hs.get_handlers().presence_handler + + if (yield presence.store.user_rooms_intersect( + [u.to_string() for u in observer_user, observed_user] + )): + defer.returnValue(True) + + if observed_user.is_mine: + pushmap = presence._local_pushmap + + defer.returnValue( + observed_user.localpart in pushmap and + observer_user in pushmap[observed_user.localpart] + ) + else: + recvmap = presence._remote_recvmap + + defer.returnValue( + observed_user in recvmap and + observer_user in recvmap[observed_user] + ) + + @defer.inlineCallbacks def get_new_events_for_user(self, user, from_key, limit): from_key = int(from_key) + observer_user = user + presence = self.hs.get_handlers().presence_handler cachemap = presence._user_cachemap - # TODO(paul): limit, and filter by visibility - updates = [(k, cachemap[k]) for k in cachemap - if from_key < cachemap[k].serial] + updates = [] + # TODO(paul): use a DeferredList ? How to limit concurrency. + for observed_user in cachemap.keys(): + if not (from_key < cachemap[observed_user].serial): + continue + + if (yield self.is_visible(observer_user, observed_user)): + updates.append((observed_user, cachemap[observed_user])) + + # TODO(paul): limit if updates: clock = self.clock @@ -789,20 +810,23 @@ class PresenceEventSource(object): latest_serial = max([x[1].serial for x in updates]) data = [x[1].make_event(user=x[0], clock=clock) for x in updates] - return ((data, latest_serial)) + defer.returnValue((data, latest_serial)) else: - return (([], presence._user_cachemap_latest_serial)) + defer.returnValue(([], presence._user_cachemap_latest_serial)) def get_current_key(self): presence = self.hs.get_handlers().presence_handler return presence._user_cachemap_latest_serial + @defer.inlineCallbacks def get_pagination_rows(self, user, pagination_config, key): # TODO (erikj): Does this make sense? Ordering? from_token = pagination_config.from_token to_token = pagination_config.to_token + observer_user = user + from_key = int(from_token.presence_key) if to_token: @@ -813,7 +837,17 @@ class PresenceEventSource(object): presence = self.hs.get_handlers().presence_handler cachemap = presence._user_cachemap - # TODO(paul): limit, and filter by visibility + updates = [] + # TODO(paul): use a DeferredList ? How to limit concurrency. + for observed_user in cachemap.keys(): + if not (to_key < cachemap[observed_user].serial < from_key): + continue + + if (yield self.is_visible(observer_user, observed_user)): + updates.append((observed_user, cachemap[observed_user])) + + # TODO(paul): limit + updates = [(k, cachemap[k]) for k in cachemap if to_key < cachemap[k].serial < from_key] @@ -831,13 +865,13 @@ class PresenceEventSource(object): next_token = next_token.copy_and_replace( "presence_key", earliest_serial ) - return ((data, next_token)) + defer.returnValue((data, next_token)) else: if not to_token: to_token = from_token.copy_and_replace( "presence_key", 0 ) - return (([], to_token)) + defer.returnValue(([], to_token)) class UserPresenceCache(object): @@ -851,7 +885,6 @@ class UserPresenceCache(object): def update(self, state, serial): assert("mtime_age" not in state) - assert("state" not in state) self.state.update(state) # Delete keys that are now 'None' @@ -869,11 +902,6 @@ class UserPresenceCache(object): def get_state(self): # clone it so caller can't break our cache state = dict(self.state) - - # Legacy handling - if "presence" in state: - state["state"] = state["presence"] - return state def make_event(self, user, clock): diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 6799132054..023d8c0cf2 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 593c603346..bee052274f 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,9 +20,13 @@ from synapse.types import UserID from synapse.api.errors import SynapseError, RegistrationError from ._base import BaseHandler import synapse.util.stringutils as stringutils +from synapse.http.client import PlainHttpClient import base64 import bcrypt +import logging + +logger = logging.getLogger(__name__) class RegistrationHandler(BaseHandler): @@ -34,7 +38,7 @@ class RegistrationHandler(BaseHandler): self.distributor.declare("registered_user") @defer.inlineCallbacks - def register(self, localpart=None, password=None): + def register(self, localpart=None, password=None, threepidCreds=None): """Registers a new client on the server. Args: @@ -47,6 +51,20 @@ class RegistrationHandler(BaseHandler): Raises: RegistrationError if there was a problem registering. """ + + if threepidCreds: + for c in threepidCreds: + logger.info("validating theeepidcred sid %s on id server %s", c['sid'], c['idServer']) + try: + threepid = yield self._threepid_from_creds(c) + except: + logger.err() + raise RegistrationError(400, "Couldn't validate 3pid") + + if not threepid: + raise RegistrationError(400, "Couldn't validate 3pid") + logger.info("got threepid medium %s address %s", threepid['medium'], threepid['address']) + password_hash = None if password: password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) @@ -61,7 +79,6 @@ class RegistrationHandler(BaseHandler): password_hash=password_hash) self.distributor.fire("registered_user", user) - defer.returnValue((user_id, token)) else: # autogen a random user ID attempts = 0 @@ -80,7 +97,6 @@ class RegistrationHandler(BaseHandler): password_hash=password_hash) self.distributor.fire("registered_user", user) - defer.returnValue((user_id, token)) except SynapseError: # if user id is taken, just generate another user_id = None @@ -90,6 +106,15 @@ class RegistrationHandler(BaseHandler): raise RegistrationError( 500, "Cannot generate user ID.") + # Now we have a matrix ID, bind it to the threepids we were given + if threepidCreds: + for c in threepidCreds: + # XXX: This should be a deferred list, shouldn't it? + yield self._bind_threepid(c, user_id) + + + defer.returnValue((user_id, token)) + def _generate_token(self, user_id): # urlsafe variant uses _ and - so use . as the separator and replace # all =s with .s so http clients don't quote =s when it is used as @@ -99,3 +124,34 @@ class RegistrationHandler(BaseHandler): def _generate_user_id(self): return "-" + stringutils.random_string(18) + + @defer.inlineCallbacks + def _threepid_from_creds(self, creds): + httpCli = PlainHttpClient(self.hs) + # XXX: make this configurable! + trustedIdServers = [ 'matrix.org:8090' ] + if not creds['idServer'] in trustedIdServers: + logger.warn('%s is not a trusted ID server: rejecting 3pid credentials', creds['idServer']) + defer.returnValue(None) + data = yield httpCli.get_json( + creds['idServer'], + "/_matrix/identity/api/v1/3pid/getValidated3pid", + { 'sid': creds['sid'], 'clientSecret': creds['clientSecret'] } + ) + + if 'medium' in data: + defer.returnValue(data) + defer.returnValue(None) + + @defer.inlineCallbacks + def _bind_threepid(self, creds, mxid): + httpCli = PlainHttpClient(self.hs) + data = yield httpCli.post_urlencoded_get_json( + creds['idServer'], + "/_matrix/identity/api/v1/3pid/bind", + { 'sid': creds['sid'], 'clientSecret': creds['clientSecret'], 'mxid':mxid } + ) + defer.returnValue(data) + + + diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 53aa77405c..8171e9eb45 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -49,6 +49,7 @@ class RoomCreationHandler(BaseRoomHandler): SynapseError if the room ID was taken, couldn't be stored, or something went horribly wrong. """ + self.ratelimit(user_id) if "room_alias_name" in config: room_alias = RoomAlias.create_local( @@ -110,8 +111,6 @@ class RoomCreationHandler(BaseRoomHandler): servers=[self.hs.hostname], ) - federation_handler = self.hs.get_handlers().federation_handler - @defer.inlineCallbacks def handle_event(event): snapshot = yield self.store.snapshot_room( @@ -138,6 +137,17 @@ class RoomCreationHandler(BaseRoomHandler): ) yield handle_event(name_event) + elif room_alias: + name = room_alias.to_string() + name_event = self.event_factory.create_event( + etype=RoomNameEvent.TYPE, + room_id=room_id, + user_id=user_id, + required_power_level=5, + content={"name": name}, + ) + + yield handle_event(name_event) if "topic" in config: topic = config["topic"] diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 3268427ecd..0ca4e5c31e 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py index 2216c0f1ca..9bff9ec169 100644 --- a/synapse/http/__init__.py +++ b/synapse/http/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/client.py b/synapse/http/client.py index 093bdf0e3f..ebf1aa47c4 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +15,8 @@ from twisted.internet import defer, reactor -from twisted.web.client import _AgentBase, _URI, readBody +from twisted.internet.error import DNSLookupError +from twisted.web.client import _AgentBase, _URI, readBody, FileBodyProducer from twisted.web.http_headers import Headers from synapse.http.endpoint import matrix_endpoint @@ -23,7 +24,9 @@ from synapse.util.async import sleep from syutil.jsonutil import encode_canonical_json -from synapse.api.errors import CodeMessageException +from synapse.api.errors import CodeMessageException, SynapseError + +from StringIO import StringIO import json import logging @@ -43,6 +46,7 @@ _destination_mappings = { class HttpClient(object): """ Interface for talking json over http """ + RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" def put_json(self, destination, path, data): """ Sends the specifed json data using PUT @@ -142,13 +146,43 @@ class TwistedHttpClient(HttpClient): destination = _destination_mappings[destination] logger.debug("get_json args: %s", args) + + retry_on_dns_fail = True + if HttpClient.RETRY_DNS_LOOKUP_FAILURES in args: + # FIXME: This isn't ideal, but the interface exposed in get_json + # isn't comprehensive enough to give caller's any control over + # their connection mechanics. + retry_on_dns_fail = args.pop(HttpClient.RETRY_DNS_LOOKUP_FAILURES) + query_bytes = urllib.urlencode(args, True) + logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) response = yield self._create_request( destination.encode("ascii"), "GET", path.encode("ascii"), - query_bytes=query_bytes + query_bytes=query_bytes, + retry_on_dns_fail=retry_on_dns_fail + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + + @defer.inlineCallbacks + def post_urlencoded_get_json(self, destination, path, args={}): + if destination in _destination_mappings: + destination = _destination_mappings[destination] + + logger.debug("post_urlencoded_get_json args: %s", args) + query_bytes = urllib.urlencode(args, True) + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + producer=FileBodyProducer(StringIO(urllib.urlencode(args))), + headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]} ) body = yield readBody(response) @@ -157,7 +191,8 @@ class TwistedHttpClient(HttpClient): @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, param_bytes=b"", - query_bytes=b"", producer=None, headers_dict={}): + query_bytes=b"", producer=None, headers_dict={}, + retry_on_dns_fail=True): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [b"Synapse"] @@ -178,10 +213,7 @@ class TwistedHttpClient(HttpClient): retries_left = 5 # TODO: setup and pass in an ssl_context to enable TLS - endpoint = matrix_endpoint( - reactor, destination, timeout=10, - ssl_context_factory=self.hs.tls_context_factory - ) + endpoint = self._getEndpoint(reactor, destination); while True: try: @@ -199,6 +231,11 @@ class TwistedHttpClient(HttpClient): logger.debug("Got response to %s", method) break except Exception as e: + if not retry_on_dns_fail and isinstance(e, DNSLookupError): + logger.warn("DNS Lookup failed to %s with %s", destination, + e) + raise SynapseError(400, "Domain specified not found.") + logger.exception("Got error in _create_request") _print_ex(e) @@ -223,6 +260,17 @@ class TwistedHttpClient(HttpClient): defer.returnValue(response) + def _getEndpoint(self, reactor, destination): + return matrix_endpoint( + reactor, destination, timeout=10, + ssl_context_factory=self.hs.tls_context_factory + ) + + +class PlainHttpClient(TwistedHttpClient): + def _getEndpoint(self, reactor, destination): + return matrix_endpoint(reactor, destination, timeout=10) + def _print_ex(e): if hasattr(e, "reasons") and e.reasons: diff --git a/synapse/http/content_repository.py b/synapse/http/content_repository.py new file mode 100644 index 0000000000..7dd4a859f8 --- /dev/null +++ b/synapse/http/content_repository.py @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .server import respond_with_json_bytes + +from synapse.util.stringutils import random_string +from synapse.api.errors import ( + cs_exception, SynapseError, CodeMessageException, Codes, cs_error +) + +from twisted.protocols.basic import FileSender +from twisted.web import server, resource +from twisted.internet import defer + +import base64 +import json +import logging +import os +import re + +logger = logging.getLogger(__name__) + + +class ContentRepoResource(resource.Resource): + """Provides file uploading and downloading. + + Uploads are POSTed to wherever this Resource is linked to. This resource + returns a "content token" which can be used to GET this content again. The + token is typically a path, but it may not be. Tokens can expire, be one-time + uses, etc. + + In this case, the token is a path to the file and contains 3 interesting + sections: + - User ID base64d (for namespacing content to each user) + - random 24 char string + - Content type base64d (so we can return it when clients GET it) + + """ + isLeaf = True + + def __init__(self, hs, directory, auth, external_addr): + resource.Resource.__init__(self) + self.hs = hs + self.directory = directory + self.auth = auth + self.external_addr = external_addr.rstrip('/') + self.max_upload_size = hs.config.max_upload_size + + if not os.path.isdir(self.directory): + os.mkdir(self.directory) + logger.info("ContentRepoResource : Created %s directory.", + self.directory) + + @defer.inlineCallbacks + def map_request_to_name(self, request): + # auth the user + auth_user = yield self.auth.get_user_by_req(request) + + # namespace all file uploads on the user + prefix = base64.urlsafe_b64encode( + auth_user.to_string() + ).replace('=', '') + + # use a random string for the main portion + main_part = random_string(24) + + # suffix with a file extension if we can make one. This is nice to + # provide a hint to clients on the file information. We will also reuse + # this info to spit back the content type to the client. + suffix = "" + if request.requestHeaders.hasHeader("Content-Type"): + content_type = request.requestHeaders.getRawHeaders( + "Content-Type")[0] + suffix = "." + base64.urlsafe_b64encode(content_type) + if (content_type.split("/")[0].lower() in + ["image", "video", "audio"]): + file_ext = content_type.split("/")[-1] + # be a little paranoid and only allow a-z + file_ext = re.sub("[^a-z]", "", file_ext) + suffix += "." + file_ext + + file_name = prefix + main_part + suffix + file_path = os.path.join(self.directory, file_name) + logger.info("User %s is uploading a file to path %s", + auth_user.to_string(), + file_path) + + # keep trying to make a non-clashing file, with a sensible max attempts + attempts = 0 + while os.path.exists(file_path): + main_part = random_string(24) + file_name = prefix + main_part + suffix + file_path = os.path.join(self.directory, file_name) + attempts += 1 + if attempts > 25: # really? Really? + raise SynapseError(500, "Unable to create file.") + + defer.returnValue(file_path) + + def render_GET(self, request): + # no auth here on purpose, to allow anyone to view, even across home + # servers. + + # TODO: A little crude here, we could do this better. + filename = request.path.split('/')[-1] + # be paranoid + filename = re.sub("[^0-9A-z.-_]", "", filename) + + file_path = self.directory + "/" + filename + + logger.debug("Searching for %s", file_path) + + if os.path.isfile(file_path): + # filename has the content type + base64_contentype = filename.split(".")[1] + content_type = base64.urlsafe_b64decode(base64_contentype) + logger.info("Sending file %s", file_path) + f = open(file_path, 'rb') + request.setHeader('Content-Type', content_type) + d = FileSender().beginFileTransfer(f, request) + + # after the file has been sent, clean up and finish the request + def cbFinished(ignored): + f.close() + request.finish() + d.addCallback(cbFinished) + else: + respond_with_json_bytes( + request, + 404, + json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)), + send_cors=True) + + return server.NOT_DONE_YET + + def render_POST(self, request): + self._async_render(request) + return server.NOT_DONE_YET + + def render_OPTIONS(self, request): + respond_with_json_bytes(request, 200, {}, send_cors=True) + return server.NOT_DONE_YET + + @defer.inlineCallbacks + def _async_render(self, request): + try: + # TODO: The checks here are a bit late. The content will have + # already been uploaded to a tmp file at this point + content_length = request.getHeader("Content-Length") + if content_length is None: + raise SynapseError( + msg="Request must specify a Content-Length", code=400 + ) + if int(content_length) > self.max_upload_size: + raise SynapseError( + msg="Upload request body is too large", + code=413, + ) + + fname = yield self.map_request_to_name(request) + + # TODO I have a suspcious feeling this is just going to block + with open(fname, "wb") as f: + f.write(request.content.read()) + + + # FIXME (erikj): These should use constants. + file_name = os.path.basename(fname) + # FIXME: we can't assume what the public mounted path of the repo is + # ...plus self-signed SSL won't work to remote clients anyway + # ...and we can't assume that it's SSL anyway, as we might want to + # server it via the non-SSL listener... + url = "%s/_matrix/content/%s" % ( + self.external_addr, file_name + ) + + respond_with_json_bytes(request, 200, + json.dumps({"content_token": url}), + send_cors=True) + + except CodeMessageException as e: + logger.exception(e) + respond_with_json_bytes(request, e.code, + json.dumps(cs_exception(e))) + except Exception as e: + logger.error("Failed to store file: %s" % e) + respond_with_json_bytes( + request, + 500, + json.dumps({"error": "Internal server error"}), + send_cors=True) + + + diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index 6c1fdcb853..7018ee3458 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/server.py b/synapse/http/server.py index 0b87718bfa..8d419c02dd 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,22 +18,16 @@ from syutil.jsonutil import ( encode_canonical_json, encode_pretty_printed_json ) from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException, Codes, cs_error + cs_exception, SynapseError, CodeMessageException ) -from synapse.util.stringutils import random_string from twisted.internet import defer, reactor -from twisted.protocols.basic import FileSender from twisted.web import server, resource from twisted.web.server import NOT_DONE_YET from twisted.web.util import redirectTo -import base64 import collections -import json import logging -import os -import re logger = logging.getLogger(__name__) @@ -140,7 +134,8 @@ class JsonResource(HttpServer, resource.Resource): self._send_response( request, e.code, - cs_exception(e) + cs_exception(e), + response_code_message=e.response_code_message ) except Exception as e: logger.exception(e) @@ -150,7 +145,8 @@ class JsonResource(HttpServer, resource.Resource): {"error": "Internal server error"} ) - def _send_response(self, request, code, response_json_object): + def _send_response(self, request, code, response_json_object, + response_code_message=None): # could alternatively use request.notifyFinish() and flip a flag when # the Deferred fires, but since the flag is RIGHT THERE it seems like # a waste. @@ -166,7 +162,8 @@ class JsonResource(HttpServer, resource.Resource): json_bytes = encode_pretty_printed_json(response_json_object) # TODO: Only enable CORS for the requests that need it. - respond_with_json_bytes(request, code, json_bytes, send_cors=True) + respond_with_json_bytes(request, code, json_bytes, send_cors=True, + response_code_message=response_code_message) @staticmethod def _request_user_agent_is_curl(request): @@ -195,162 +192,8 @@ class RootRedirect(resource.Resource): return resource.Resource.getChild(self, name, request) -class ContentRepoResource(resource.Resource): - """Provides file uploading and downloading. - - Uploads are POSTed to wherever this Resource is linked to. This resource - returns a "content token" which can be used to GET this content again. The - token is typically a path, but it may not be. Tokens can expire, be one-time - uses, etc. - - In this case, the token is a path to the file and contains 3 interesting - sections: - - User ID base64d (for namespacing content to each user) - - random 24 char string - - Content type base64d (so we can return it when clients GET it) - - """ - isLeaf = True - - def __init__(self, hs, directory, auth): - resource.Resource.__init__(self) - self.hs = hs - self.directory = directory - self.auth = auth - - if not os.path.isdir(self.directory): - os.mkdir(self.directory) - logger.info("ContentRepoResource : Created %s directory.", - self.directory) - - @defer.inlineCallbacks - def map_request_to_name(self, request): - # auth the user - auth_user = yield self.auth.get_user_by_req(request) - - # namespace all file uploads on the user - prefix = base64.urlsafe_b64encode( - auth_user.to_string() - ).replace('=', '') - - # use a random string for the main portion - main_part = random_string(24) - - # suffix with a file extension if we can make one. This is nice to - # provide a hint to clients on the file information. We will also reuse - # this info to spit back the content type to the client. - suffix = "" - if request.requestHeaders.hasHeader("Content-Type"): - content_type = request.requestHeaders.getRawHeaders( - "Content-Type")[0] - suffix = "." + base64.urlsafe_b64encode(content_type) - if (content_type.split("/")[0].lower() in - ["image", "video", "audio"]): - file_ext = content_type.split("/")[-1] - # be a little paranoid and only allow a-z - file_ext = re.sub("[^a-z]", "", file_ext) - suffix += "." + file_ext - - file_name = prefix + main_part + suffix - file_path = os.path.join(self.directory, file_name) - logger.info("User %s is uploading a file to path %s", - auth_user.to_string(), - file_path) - - # keep trying to make a non-clashing file, with a sensible max attempts - attempts = 0 - while os.path.exists(file_path): - main_part = random_string(24) - file_name = prefix + main_part + suffix - file_path = os.path.join(self.directory, file_name) - attempts += 1 - if attempts > 25: # really? Really? - raise SynapseError(500, "Unable to create file.") - - defer.returnValue(file_path) - - def render_GET(self, request): - # no auth here on purpose, to allow anyone to view, even across home - # servers. - - # TODO: A little crude here, we could do this better. - filename = request.path.split('/')[-1] - # be paranoid - filename = re.sub("[^0-9A-z.-_]", "", filename) - - file_path = self.directory + "/" + filename - - logger.debug("Searching for %s", file_path) - - if os.path.isfile(file_path): - # filename has the content type - base64_contentype = filename.split(".")[1] - content_type = base64.urlsafe_b64decode(base64_contentype) - logger.info("Sending file %s", file_path) - f = open(file_path, 'rb') - request.setHeader('Content-Type', content_type) - d = FileSender().beginFileTransfer(f, request) - - # after the file has been sent, clean up and finish the request - def cbFinished(ignored): - f.close() - request.finish() - d.addCallback(cbFinished) - else: - respond_with_json_bytes( - request, - 404, - json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)), - send_cors=True) - - return server.NOT_DONE_YET - - def render_POST(self, request): - self._async_render(request) - return server.NOT_DONE_YET - - def render_OPTIONS(self, request): - respond_with_json_bytes(request, 200, {}, send_cors=True) - return server.NOT_DONE_YET - - @defer.inlineCallbacks - def _async_render(self, request): - try: - fname = yield self.map_request_to_name(request) - - # TODO I have a suspcious feeling this is just going to block - with open(fname, "wb") as f: - f.write(request.content.read()) - - - # FIXME (erikj): These should use constants. - file_name = os.path.basename(fname) - # FIXME: we can't assume what the public mounted path of the repo is - # ...plus self-signed SSL won't work to remote clients anyway - # ...and we can't assume that it's SSL anyway, as we might want to - # server it via the non-SSL listener... - url = "https://%s/_matrix/content/%s" % ( - self.hs.domain_with_port, file_name - ) - - respond_with_json_bytes(request, 200, - json.dumps({"content_token": url}), - send_cors=True) - - except CodeMessageException as e: - logger.exception(e) - respond_with_json_bytes(request, e.code, - json.dumps(cs_exception(e))) - except Exception as e: - logger.error("Failed to store file: %s" % e) - respond_with_json_bytes( - request, - 500, - json.dumps({"error": "Internal server error"}), - send_cors=True) - - -def respond_with_json_bytes(request, code, json_bytes, send_cors=False): +def respond_with_json_bytes(request, code, json_bytes, send_cors=False, + response_code_message=None): """Sends encoded JSON in response to the given request. Args: @@ -362,7 +205,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False): Returns: twisted.web.server.NOT_DONE_YET""" - request.setResponseCode(code) + request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") if send_cors: diff --git a/synapse/notifier.py b/synapse/notifier.py index 3260aa744f..5b02c71d1e 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -167,7 +167,12 @@ class Notifier(object): ) def eb(failure): - logger.exception("Failed to notify listener", failure) + logger.error("Failed to notify listener", + exc_info=( + failure.type, + failure.value, + failure.getTracebackObject()) + ) yield defer.DeferredList( [notify(l).addErrback(eb) for l in listeners] diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index f33024e72a..ed785cfbd5 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/base.py b/synapse/rest/base.py index e855d293e5..2e8e3fa7d4 100644 --- a/synapse/rest/base.py +++ b/synapse/rest/base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py index dc347652a0..18df7c8d8b 100644 --- a/synapse/rest/directory.py +++ b/synapse/rest/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ from twisted.internet import defer +from synapse.api.errors import SynapseError, Codes from base import RestServlet, client_path_pattern import json @@ -44,8 +45,10 @@ class ClientDirectoryServer(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - # TODO(erikj): Exceptions - content = json.loads(request.content.read()) + content = _parse_json(request) + if not "room_id" in content: + raise SynapseError(400, "Missing room_id key", + errcode=Codes.BAD_JSON) logger.debug("Got content: %s", content) @@ -54,7 +57,7 @@ class ClientDirectoryServer(RestServlet): logger.debug("Got room name: %s", room_alias.to_string()) room_id = content["room_id"] - servers = content["servers"] + servers = content["servers"] if "servers" in content else None logger.debug("Got room_id: %s", room_id) logger.debug("Got servers: %s", servers) @@ -68,7 +71,20 @@ class ClientDirectoryServer(RestServlet): yield dir_handler.create_association( room_alias, room_id, servers ) + except SynapseError as e: + raise e except: logger.exception("Failed to create association") defer.returnValue((200, {})) + + +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) diff --git a/synapse/rest/events.py b/synapse/rest/events.py index 2e7563d14b..7fde143200 100644 --- a/synapse/rest/events.py +++ b/synapse/rest/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/initial_sync.py b/synapse/rest/initial_sync.py index d18c4c0f60..a1cb442256 100644 --- a/synapse/rest/initial_sync.py +++ b/synapse/rest/initial_sync.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/login.py b/synapse/rest/login.py index 99e4f10aac..c7bf901c8e 100644 --- a/synapse/rest/login.py +++ b/synapse/rest/login.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/presence.py b/synapse/rest/presence.py index bce3943542..7fc8ce4404 100644 --- a/synapse/rest/presence.py +++ b/synapse/rest/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,11 +17,12 @@ """ from twisted.internet import defer +from synapse.api.errors import SynapseError from base import RestServlet, client_path_pattern import json import logging - +import urllib logger = logging.getLogger(__name__) @@ -32,6 +33,7 @@ class PresenceStatusRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) + user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) state = yield self.handlers.presence_handler.get_state( @@ -42,25 +44,26 @@ class PresenceStatusRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) + user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) state = {} try: content = json.loads(request.content.read()) - # Legacy handling - if "state" in content: - state["presence"] = content.pop("state") - else: - state["presence"] = content.pop("presence") + state["presence"] = content.pop("presence") if "status_msg" in content: state["status_msg"] = content.pop("status_msg") + if not isinstance(state["status_msg"], basestring): + raise SynapseError(400, "status_msg must be a string.") if content: raise KeyError() + except SynapseError as e: + raise e except: - defer.returnValue((400, "Unable to parse state")) + raise SynapseError(400, "Unable to parse state") yield self.handlers.presence_handler.set_state( target_user=user, auth_user=auth_user, state=state) @@ -77,13 +80,14 @@ class PresenceListRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) + user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) if not user.is_mine: - defer.returnValue((400, "User not hosted on this Home Server")) + raise SynapseError(400, "User not hosted on this Home Server") if auth_user != user: - defer.returnValue((400, "Cannot get another user's presence list")) + raise SynapseError(400, "Cannot get another user's presence list") presence = yield self.handlers.presence_handler.get_presence_list( observer_user=user, accepted=True) @@ -97,31 +101,40 @@ class PresenceListRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) + user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) if not user.is_mine: - defer.returnValue((400, "User not hosted on this Home Server")) + raise SynapseError(400, "User not hosted on this Home Server") if auth_user != user: - defer.returnValue(( - 400, "Cannot modify another user's presence list")) + raise SynapseError( + 400, "Cannot modify another user's presence list") try: content = json.loads(request.content.read()) except: logger.exception("JSON parse error") - defer.returnValue((400, "Unable to parse content")) + raise SynapseError(400, "Unable to parse content") deferreds = [] if "invite" in content: for u in content["invite"]: + if not isinstance(u, basestring): + raise SynapseError(400, "Bad invite value.") + if len(u) == 0: + continue invited_user = self.hs.parse_userid(u) deferreds.append(self.handlers.presence_handler.send_invite( observer_user=user, observed_user=invited_user)) if "drop" in content: for u in content["drop"]: + if not isinstance(u, basestring): + raise SynapseError(400, "Bad drop value.") + if len(u) == 0: + continue dropped_user = self.hs.parse_userid(u) deferreds.append(self.handlers.presence_handler.drop( observer_user=user, observed_user=dropped_user)) diff --git a/synapse/rest/profile.py b/synapse/rest/profile.py index 06076667c7..2e17f87fa1 100644 --- a/synapse/rest/profile.py +++ b/synapse/rest/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ from twisted.internet import defer from base import RestServlet, client_path_pattern import json +import urllib class ProfileDisplaynameRestServlet(RestServlet): @@ -26,6 +27,7 @@ class ProfileDisplaynameRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): + user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) displayname = yield self.handlers.profile_handler.get_displayname( @@ -37,6 +39,7 @@ class ProfileDisplaynameRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) + user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) try: @@ -59,6 +62,7 @@ class ProfileAvatarURLRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): + user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) avatar_url = yield self.handlers.profile_handler.get_avatar_url( @@ -70,6 +74,7 @@ class ProfileAvatarURLRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) + user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) try: @@ -92,6 +97,7 @@ class ProfileRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): + user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) displayname = yield self.handlers.profile_handler.get_displayname( diff --git a/synapse/rest/register.py b/synapse/rest/register.py index f17ec11cf4..b8de3b250d 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -47,10 +47,15 @@ class RegisterRestServlet(RestServlet): except KeyError: pass # user_id is optional + threepidCreds = None + if 'threepidCreds' in register_json: + threepidCreds = register_json['threepidCreds'] + handler = self.handlers.registration_handler (user_id, token) = yield handler.register( localpart=desired_user_id, - password=password) + password=password, + threepidCreds=threepidCreds) result = { "user_id": user_id, diff --git a/synapse/rest/room.py b/synapse/rest/room.py index a10b3b54f9..308b447090 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -388,7 +388,7 @@ class RoomMembershipRestServlet(RestServlet): def register(self, http_server): # /rooms/$roomid/[invite|join|leave] PATTERN = ("/rooms/(?P<room_id>[^/]*)/" + - "(?P<membership_action>join|invite|leave)") + "(?P<membership_action>join|invite|leave|ban|kick)") register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks @@ -399,11 +399,14 @@ class RoomMembershipRestServlet(RestServlet): # target user is you unless it is an invite state_key = user.to_string() - if membership_action == "invite": + if membership_action in ["invite", "ban", "kick"]: if "user_id" not in content: raise SynapseError(400, "Missing user_id key.") state_key = content["user_id"] + if membership_action == "kick": + membership_action = "leave" + event = self.event_factory.create_event( etype=RoomMemberEvent.TYPE, content={"membership": unicode(membership_action)}, diff --git a/synapse/rest/transactions.py b/synapse/rest/transactions.py index b8aa1ef11c..e06dcc8c57 100644 --- a/synapse/rest/transactions.py +++ b/synapse/rest/transactions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/server.py b/synapse/server.py index 3e72b2bcd5..83368ea5a7 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,6 +32,7 @@ from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager from synapse.streams.events import EventSources +from synapse.api.ratelimiting import Ratelimiter class BaseHomeServer(object): @@ -73,6 +74,7 @@ class BaseHomeServer(object): 'resource_for_web_client', 'resource_for_content_repo', 'event_sources', + 'ratelimiter', ] def __init__(self, hostname, **kwargs): @@ -190,6 +192,9 @@ class HomeServer(BaseHomeServer): def build_event_sources(self): return EventSources(self) + def build_ratelimiter(self): + return Ratelimiter() + def register_servlets(self): """ Register all servlets associated with this HomeServer. """ diff --git a/synapse/state.py b/synapse/state.py index e1a1a159bb..36d8210eb5 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -179,6 +179,18 @@ class StateHandler(object): key=lambda x: x.depth ) + if not hasattr(missing_prev, "prev_state_id"): + # FIXME Hmm + # temporary fallback + for algo in conflict_res: + new_res, curr_res = algo(new_branch, current_branch) + + if new_res < curr_res: + defer.returnValue(False) + elif new_res > curr_res: + defer.returnValue(True) + return + pdu_id = missing_prev.prev_state_id origin = missing_prev.prev_state_origin diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index aadaab06e7..d97014f4da 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 33d56f47ce..bae50e7d1f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -79,19 +79,21 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. - def _simple_insert(self, table, values): + def _simple_insert(self, table, values, or_replace=False): """Executes an INSERT query on the named table. Args: table : string giving the table name values : dict of new column names and values for them + or_replace : bool; if True performs an INSERT OR REPLACE """ return self._db_pool.runInteraction( - self._simple_insert_txn, table, values, + self._simple_insert_txn, table, values, or_replace=or_replace ) - def _simple_insert_txn(self, txn, table, values): - sql = "INSERT INTO %s (%s) VALUES(%s)" % ( + def _simple_insert_txn(self, txn, table, values, or_replace=False): + sql = "%s INTO %s (%s) VALUES(%s)" % ( + ("INSERT OR REPLACE" if or_replace else "INSERT"), table, ", ".join(k for k in values), ", ".join("?" for k in values) diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index b22ce02f3f..bf55449253 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py index bac3dea955..8a18617188 100644 --- a/synapse/storage/feedback.py +++ b/synapse/storage/feedback.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 4d19b9f641..5a38c3e8f2 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 9fd44f2454..0bf97e37ee 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index a529104f4d..71b2bb084d 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 91dd565033..7e1fdd9d88 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index b1e4196435..fd762bc643 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 01ae190316..017169ce00 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 2746126e85..75c9a60101 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -165,7 +165,7 @@ class RoomMemberStore(SQLBaseStore): defer.returnValue(results) @defer.inlineCallbacks - def do_users_share_a_room(self, user_list): + def user_rooms_intersect(self, user_list): """ Checks whether a list of users share a room. """ user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_list)) diff --git a/synapse/storage/schema/delta/v2.sql b/synapse/storage/schema/delta/v2.sql new file mode 100644 index 0000000000..73b140465e --- /dev/null +++ b/synapse/storage/schema/delta/v2.sql @@ -0,0 +1,168 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS events( + stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, + topological_ordering INTEGER NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, + content TEXT NOT NULL, + unrecognized_keys TEXT, + processed BOOL NOT NULL, + outlier BOOL NOT NULL, + CONSTRAINT ev_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); +CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); +CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); +CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); + +CREATE TABLE IF NOT EXISTS state_events( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT +); + +CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id); +CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); +CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); +CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); + + +CREATE TABLE IF NOT EXISTS current_state_events( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); +CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); +CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); + +CREATE TABLE IF NOT EXISTS room_memberships( + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); +CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); + +CREATE TABLE IF NOT EXISTS feedback( + event_id TEXT NOT NULL, + feedback_type TEXT, + target_event_id TEXT, + sender TEXT, + room_id TEXT +); + +CREATE TABLE IF NOT EXISTS topics( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + topic TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS room_names( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + name TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS rooms( + room_id TEXT PRIMARY KEY NOT NULL, + is_public INTEGER, + creator TEXT +); + +CREATE TABLE IF NOT EXISTS room_join_rules( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + join_rule TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS room_join_rules_event_id ON room_join_rules(event_id); +CREATE INDEX IF NOT EXISTS room_join_rules_room_id ON room_join_rules(room_id); + + +CREATE TABLE IF NOT EXISTS room_power_levels( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + user_id TEXT NOT NULL, + level INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS room_power_levels_event_id ON room_power_levels(event_id); +CREATE INDEX IF NOT EXISTS room_power_levels_room_id ON room_power_levels(room_id); +CREATE INDEX IF NOT EXISTS room_power_levels_room_user ON room_power_levels(room_id, user_id); + + +CREATE TABLE IF NOT EXISTS room_default_levels( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + level INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS room_default_levels_event_id ON room_default_levels(event_id); +CREATE INDEX IF NOT EXISTS room_default_levels_room_id ON room_default_levels(room_id); + + +CREATE TABLE IF NOT EXISTS room_add_state_levels( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + level INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS room_add_state_levels_event_id ON room_add_state_levels(event_id); +CREATE INDEX IF NOT EXISTS room_add_state_levels_room_id ON room_add_state_levels(room_id); + + +CREATE TABLE IF NOT EXISTS room_send_event_levels( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + level INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS room_send_event_levels_event_id ON room_send_event_levels(event_id); +CREATE INDEX IF NOT EXISTS room_send_event_levels_room_id ON room_send_event_levels(room_id); + + +CREATE TABLE IF NOT EXISTS room_ops_levels( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + ban_level INTEGER, + kick_level INTEGER +); + +CREATE INDEX IF NOT EXISTS room_ops_levels_event_id ON room_ops_levels(event_id); +CREATE INDEX IF NOT EXISTS room_ops_levels_room_id ON room_ops_levels(room_id); + + +CREATE TABLE IF NOT EXISTS room_hosts( + room_id TEXT NOT NULL, + host TEXT NOT NULL, + CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE +); + +CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); + +PRAGMA user_version = 2; diff --git a/synapse/storage/schema/edge_pdus.sql b/synapse/storage/schema/edge_pdus.sql index 17b3c52f0d..8a00868065 100644 --- a/synapse/storage/schema/edge_pdus.sql +++ b/synapse/storage/schema/edge_pdus.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 matrix.org +/* Copyright 2014 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index dbefbbda31..6ffea51310 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 matrix.org +/* Copyright 2014 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/keys.sql b/synapse/storage/schema/keys.sql index 45cdbcecae..706a1a03ff 100644 --- a/synapse/storage/schema/keys.sql +++ b/synapse/storage/schema/keys.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 matrix.org +/* Copyright 2014 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/pdu.sql b/synapse/storage/schema/pdu.sql index ca3de005e9..16e111a56c 100644 --- a/synapse/storage/schema/pdu.sql +++ b/synapse/storage/schema/pdu.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 matrix.org +/* Copyright 2014 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/presence.sql b/synapse/storage/schema/presence.sql index b1081d3aab..595b3b5a69 100644 --- a/synapse/storage/schema/presence.sql +++ b/synapse/storage/schema/presence.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 matrix.org +/* Copyright 2014 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/profiles.sql b/synapse/storage/schema/profiles.sql index 1092d7672c..58209f1af0 100644 --- a/synapse/storage/schema/profiles.sql +++ b/synapse/storage/schema/profiles.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 matrix.org +/* Copyright 2014 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/room_aliases.sql b/synapse/storage/schema/room_aliases.sql index 71a8b90e4d..9191016814 100644 --- a/synapse/storage/schema/room_aliases.sql +++ b/synapse/storage/schema/room_aliases.sql @@ -1,3 +1,18 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + CREATE TABLE IF NOT EXISTS room_aliases( room_alias TEXT NOT NULL, room_id TEXT NOT NULL diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index 4b1a2368f6..88e3e4e04d 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 matrix.org +/* Copyright 2014 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/schema/users.sql b/synapse/storage/schema/users.sql index 46b60297cb..2519702971 100644 --- a/synapse/storage/schema/users.sql +++ b/synapse/storage/schema/users.sql @@ -1,4 +1,4 @@ -/* Copyright 2014 matrix.org +/* Copyright 2014 OpenMarket Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 0b78222827..2cb0067a67 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index a277e4971a..7467e1035b 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/__init__.py b/synapse/streams/__init__.py index fe8a073cd3..f9811bfa04 100644 --- a/synapse/streams/__init__.py +++ b/synapse/streams/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/config.py b/synapse/streams/config.py index 01bab568ff..6483ce2e25 100644 --- a/synapse/streams/config.py +++ b/synapse/streams/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/streams/events.py b/synapse/streams/events.py index 08d6e6f733..41715436b0 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/types.py b/synapse/types.py index 1a9dceabf5..c51bc8e4f2 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 3ea431a7f9..c9a73b0413 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/async.py b/synapse/util/async.py index ebbdc00ae1..647ea6142c 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 9605d7d1b9..1de50e049f 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,7 +32,9 @@ class Distributor(object): model will do for today. """ - def __init__(self): + def __init__(self, suppress_failures=True): + self.suppress_failures = suppress_failures + self.signals = {} self.pre_registration = {} @@ -40,7 +42,9 @@ class Distributor(object): if name in self.signals: raise KeyError("%r already has a signal named %s" % (self, name)) - self.signals[name] = Signal(name) + self.signals[name] = Signal(name, + suppress_failures=self.suppress_failures, + ) if name in self.pre_registration: signal = self.signals[name] @@ -74,8 +78,9 @@ class Signal(object): method into all of the observers. """ - def __init__(self, name): + def __init__(self, name, suppress_failures): self.name = name + self.suppress_failures = suppress_failures self.observers = [] def observe(self, observer): @@ -104,6 +109,10 @@ class Signal(object): failure.type, failure.value, failure.getTracebackObject())) + if not self.suppress_failures: + raise failure deferreds.append(d.addErrback(eb)) - return defer.DeferredList(deferreds) + return defer.DeferredList( + deferreds, fireOnOneErrback=not self.suppress_failures + ) diff --git a/synapse/util/jsonobject.py b/synapse/util/jsonobject.py index e2840b59f9..6c99705747 100644 --- a/synapse/util/jsonobject.py +++ b/synapse/util/jsonobject.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/lockutils.py b/synapse/util/lockutils.py index d0bb50d035..3a84c09db4 100644 --- a/synapse/util/lockutils.py +++ b/synapse/util/lockutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index b94a749786..fadf0bd510 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ from functools import wraps import logging import inspect -import traceback def log_function(f): diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index e1b0796e56..8767e437dd 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. |