From 6e18805ac2906ea52d2374024392332af1a603b7 Mon Sep 17 00:00:00 2001 From: Glyph Date: Sun, 11 Dec 2016 01:44:02 -0800 Subject: IPv6 support for client.py This is an (untested) general sketch of how to use wrapClientTLS to implement TLS over IPv6, as well as faster connections over IPv4. --- synapse/http/client.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) (limited to 'synapse') diff --git a/synapse/http/client.py b/synapse/http/client.py index 3ec9bc7faf..c60e3c2ac0 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -386,26 +386,21 @@ class SpiderEndpointFactory(object): def endpointForURI(self, uri): logger.info("Getting endpoint for %s", uri.toBytes()) + if uri.scheme == "http": - return SpiderEndpoint( - reactor, uri.host, uri.port, self.blacklist, self.whitelist, - endpoint=TCP4ClientEndpoint, - endpoint_kw_args={ - 'timeout': 15 - }, - ) + endpoint_factory = HostnameEndpoint elif uri.scheme == "https": - tlsPolicy = self.policyForHTTPS.creatorForNetloc(uri.host, uri.port) - return SpiderEndpoint( - reactor, uri.host, uri.port, self.blacklist, self.whitelist, - endpoint=SSL4ClientEndpoint, - endpoint_kw_args={ - 'sslContextFactory': tlsPolicy, - 'timeout': 15 - }, - ) + tlsCreator = self.policyForHTTPS.creatorForNetloc(uri.host, uri.port) + def endpoint_factory(reactor, host, port, **kw): + return wrapClientTLS(tlsCreator, HostnameEndpoint(reactor, host, port, **kw) else: logger.warn("Can't get endpoint for unrecognised scheme %s", uri.scheme) + return None + return SpiderEndpoint( + reactor, uri.host, uri.port, self.blacklist, self.whitelist, + endpoint=endpoint_factory, endpoint_kw_args=dict(timeout=15), + ) + class SpiderHttpClient(SimpleHttpClient): -- cgit 1.4.1 From 9f07f4c5595b0eff19c9740c44803700b01b14af Mon Sep 17 00:00:00 2001 From: Glyph Date: Sun, 11 Dec 2016 01:46:43 -0800 Subject: IPv6 support for endpoint.py Similar to https://github.com/matrix-org/synapse/pull/1689, but for endpoint.py --- synapse/http/endpoint.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index 442696d393..5e2e428dbf 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint +from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.internet import defer from twisted.internet.error import ConnectError from twisted.names import client, dns @@ -58,11 +58,11 @@ def matrix_federation_endpoint(reactor, destination, ssl_context_factory=None, endpoint_kw_args.update(timeout=timeout) if ssl_context_factory is None: - transport_endpoint = TCP4ClientEndpoint + transport_endpoint = HostnameEndpoint default_port = 8008 else: - transport_endpoint = SSL4ClientEndpoint - endpoint_kw_args.update(sslContextFactory=ssl_context_factory) + def transport_endpoint(reactor, host, port): + return wrapClientTLS(ssl_context_factory, HostnameEndpoint(reactor, host, port)) default_port = 8448 if port is None: -- cgit 1.4.1 From d3bd94805f6ef68e75d8c2e39b8c97ea5ce88286 Mon Sep 17 00:00:00 2001 From: Johannes Löthberg Date: Mon, 12 Dec 2016 16:19:54 +0100 Subject: Fixup for #1689 and #1690 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johannes Löthberg --- synapse/http/client.py | 11 +++++++---- synapse/http/endpoint.py | 10 ++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) (limited to 'synapse') diff --git a/synapse/http/client.py b/synapse/http/client.py index c60e3c2ac0..37988716e7 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -25,7 +25,7 @@ from synapse.http.endpoint import SpiderEndpoint from canonicaljson import encode_canonical_json from twisted.internet import defer, reactor, ssl, protocol, task -from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint +from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.web.client import ( BrowserLikeRedirectAgent, ContentDecoderAgent, GzipDecoder, Agent, readBody, PartialDownloadError, @@ -386,13 +386,16 @@ class SpiderEndpointFactory(object): def endpointForURI(self, uri): logger.info("Getting endpoint for %s", uri.toBytes()) - + if uri.scheme == "http": endpoint_factory = HostnameEndpoint elif uri.scheme == "https": tlsCreator = self.policyForHTTPS.creatorForNetloc(uri.host, uri.port) + def endpoint_factory(reactor, host, port, **kw): - return wrapClientTLS(tlsCreator, HostnameEndpoint(reactor, host, port, **kw) + return wrapClientTLS( + tlsCreator, + HostnameEndpoint(reactor, host, port, **kw)) else: logger.warn("Can't get endpoint for unrecognised scheme %s", uri.scheme) return None @@ -400,7 +403,7 @@ class SpiderEndpointFactory(object): reactor, uri.host, uri.port, self.blacklist, self.whitelist, endpoint=endpoint_factory, endpoint_kw_args=dict(timeout=15), ) - + class SpiderHttpClient(SimpleHttpClient): diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index 5e2e428dbf..1c17a28406 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -61,8 +61,10 @@ def matrix_federation_endpoint(reactor, destination, ssl_context_factory=None, transport_endpoint = HostnameEndpoint default_port = 8008 else: - def transport_endpoint(reactor, host, port): - return wrapClientTLS(ssl_context_factory, HostnameEndpoint(reactor, host, port)) + def transport_endpoint(reactor, host, port, timeout): + return wrapClientTLS( + ssl_context_factory, + HostnameEndpoint(reactor, host, port, timeout=timeout)) default_port = 8448 if port is None: @@ -80,7 +82,7 @@ class SpiderEndpoint(object): Implements twisted.internet.interfaces.IStreamClientEndpoint. """ def __init__(self, reactor, host, port, blacklist, whitelist, - endpoint=TCP4ClientEndpoint, endpoint_kw_args={}): + endpoint=HostnameEndpoint, endpoint_kw_args={}): self.reactor = reactor self.host = host self.port = port @@ -118,7 +120,7 @@ class SRVClientEndpoint(object): """ def __init__(self, reactor, service, domain, protocol="tcp", - default_port=None, endpoint=TCP4ClientEndpoint, + default_port=None, endpoint=HostnameEndpoint, endpoint_kw_args={}): self.reactor = reactor self.service_name = "_%s._%s.%s" % (service, protocol, domain) -- cgit 1.4.1 From 0648e76979e4626cf3719edc5958eb4f170e0d1e Mon Sep 17 00:00:00 2001 From: Johannes Löthberg Date: Mon, 12 Dec 2016 18:40:39 +0100 Subject: Remove spurious newline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apparently I just removed the spaces instead... Signed-off-by: Johannes Löthberg --- synapse/http/client.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse') diff --git a/synapse/http/client.py b/synapse/http/client.py index 37988716e7..ca2f770f5d 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -405,7 +405,6 @@ class SpiderEndpointFactory(object): ) - class SpiderHttpClient(SimpleHttpClient): """ Separate HTTP client for spidering arbitrary URLs. -- cgit 1.4.1 From b2f8642d3df30c88704e02422325b394880f66eb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Dec 2016 16:11:43 +0000 Subject: Cache network room list queries. --- synapse/handlers/room_list.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 667223df0c..19eebbd43f 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -62,17 +62,18 @@ class RoomListHandler(BaseHandler): appservice and network id to use an appservice specific one. Setting to None returns all public rooms across all lists. """ - if search_filter or (network_tuple and network_tuple.appservice_id is not None): + if search_filter: # We explicitly don't bother caching searches or requests for # appservice specific lists. return self._get_public_room_list( limit, since_token, search_filter, network_tuple=network_tuple, ) - result = self.response_cache.get((limit, since_token)) + key = (limit, since_token, network_tuple) + result = self.response_cache.get(key) if not result: result = self.response_cache.set( - (limit, since_token), + key, self._get_public_room_list( limit, since_token, network_tuple=network_tuple ) -- cgit 1.4.1 From 7dfd70fc834a14b7003beb220eebae6fead5dbf3 Mon Sep 17 00:00:00 2001 From: Johannes Löthberg Date: Sun, 18 Dec 2016 20:42:43 +0100 Subject: Add support for specifying multiple bind addresses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johannes Löthberg --- synapse/app/appservice.py | 54 +++++++++++++++++----------- synapse/app/client_reader.py | 54 +++++++++++++++++----------- synapse/app/federation_reader.py | 54 +++++++++++++++++----------- synapse/app/federation_sender.py | 54 +++++++++++++++++----------- synapse/app/homeserver.py | 76 ++++++++++++++++++++++++---------------- synapse/app/media_repository.py | 54 +++++++++++++++++----------- synapse/app/pusher.py | 65 +++++++++++++++++++++++----------- synapse/app/synchrotron.py | 54 +++++++++++++++++----------- 8 files changed, 294 insertions(+), 171 deletions(-) (limited to 'synapse') diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index dd9ee406a1..e24c1e1eda 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -76,7 +76,8 @@ class AppserviceServer(HomeServer): def _listen_http(self, listener_config): port = listener_config["port"] - bind_address = listener_config.get("bind_address", "") + bind_address = listener_config.get("bind_address", None) + bind_addresses = listener_config.get("bind_addresses", []) site_tag = listener_config.get("tag", port) resources = {} for res in listener_config["resources"]: @@ -85,16 +86,22 @@ class AppserviceServer(HomeServer): resources[METRICS_PREFIX] = MetricsResource(self) root_resource = create_resource_tree(resources, Resource()) - reactor.listenTCP( - port, - SynapseSite( - "synapse.access.http.%s" % (site_tag,), - site_tag, - listener_config, - root_resource, - ), - interface=bind_address - ) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + port, + SynapseSite( + "synapse.access.http.%s" % (site_tag,), + site_tag, + listener_config, + root_resource, + ), + interface=address + ) + logger.info("Synapse appservice now listening on port %d", port) def start_listening(self, listeners): @@ -102,15 +109,22 @@ class AppserviceServer(HomeServer): if listener["type"] == "http": self._listen_http(listener) elif listener["type"] == "manhole": - reactor.listenTCP( - listener["port"], - manhole( - username="matrix", - password="rabbithole", - globals={"hs": self}, - ), - interface=listener.get("bind_address", '127.0.0.1') - ) + bind_address = listener.get("bind_address", None) + bind_addresses = listener.get("bind_addresses", []) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + listener["port"], + manhole( + username="matrix", + password="rabbithole", + globals={"hs": self}, + ), + interface=address + ) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 0086a2977e..305a82b664 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -90,7 +90,8 @@ class ClientReaderServer(HomeServer): def _listen_http(self, listener_config): port = listener_config["port"] - bind_address = listener_config.get("bind_address", "") + bind_address = listener_config.get("bind_address", None) + bind_addresses = listener_config.get("bind_addresses", []) site_tag = listener_config.get("tag", port) resources = {} for res in listener_config["resources"]: @@ -108,16 +109,22 @@ class ClientReaderServer(HomeServer): }) root_resource = create_resource_tree(resources, Resource()) - reactor.listenTCP( - port, - SynapseSite( - "synapse.access.http.%s" % (site_tag,), - site_tag, - listener_config, - root_resource, - ), - interface=bind_address - ) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + port, + SynapseSite( + "synapse.access.http.%s" % (site_tag,), + site_tag, + listener_config, + root_resource, + ), + interface=address + ) + logger.info("Synapse client reader now listening on port %d", port) def start_listening(self, listeners): @@ -125,15 +132,22 @@ class ClientReaderServer(HomeServer): if listener["type"] == "http": self._listen_http(listener) elif listener["type"] == "manhole": - reactor.listenTCP( - listener["port"], - manhole( - username="matrix", - password="rabbithole", - globals={"hs": self}, - ), - interface=listener.get("bind_address", '127.0.0.1') - ) + bind_address = listener.get("bind_address", None) + bind_addresses = listener.get("bind_addresses", []) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + listener["port"], + manhole( + username="matrix", + password="rabbithole", + globals={"hs": self}, + ), + interface=address + ) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index b5f59a9931..321dfc7cd5 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -86,7 +86,8 @@ class FederationReaderServer(HomeServer): def _listen_http(self, listener_config): port = listener_config["port"] - bind_address = listener_config.get("bind_address", "") + bind_address = listener_config.get("bind_address", None) + bind_addresses = listener_config.get("bind_addresses", []) site_tag = listener_config.get("tag", port) resources = {} for res in listener_config["resources"]: @@ -99,16 +100,22 @@ class FederationReaderServer(HomeServer): }) root_resource = create_resource_tree(resources, Resource()) - reactor.listenTCP( - port, - SynapseSite( - "synapse.access.http.%s" % (site_tag,), - site_tag, - listener_config, - root_resource, - ), - interface=bind_address - ) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + port, + SynapseSite( + "synapse.access.http.%s" % (site_tag,), + site_tag, + listener_config, + root_resource, + ), + interface=address + ) + logger.info("Synapse federation reader now listening on port %d", port) def start_listening(self, listeners): @@ -116,15 +123,22 @@ class FederationReaderServer(HomeServer): if listener["type"] == "http": self._listen_http(listener) elif listener["type"] == "manhole": - reactor.listenTCP( - listener["port"], - manhole( - username="matrix", - password="rabbithole", - globals={"hs": self}, - ), - interface=listener.get("bind_address", '127.0.0.1') - ) + bind_address = listener.get("bind_address", None) + bind_addresses = listener.get("bind_addresses", []) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + listener["port"], + manhole( + username="matrix", + password="rabbithole", + globals={"hs": self}, + ), + interface=address + ) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index 80ea4c8062..8092fd316c 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -82,7 +82,8 @@ class FederationSenderServer(HomeServer): def _listen_http(self, listener_config): port = listener_config["port"] - bind_address = listener_config.get("bind_address", "") + bind_address = listener_config.get("bind_address", None) + bind_addresses = listener_config.get("bind_addresses", []) site_tag = listener_config.get("tag", port) resources = {} for res in listener_config["resources"]: @@ -91,16 +92,22 @@ class FederationSenderServer(HomeServer): resources[METRICS_PREFIX] = MetricsResource(self) root_resource = create_resource_tree(resources, Resource()) - reactor.listenTCP( - port, - SynapseSite( - "synapse.access.http.%s" % (site_tag,), - site_tag, - listener_config, - root_resource, - ), - interface=bind_address - ) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + port, + SynapseSite( + "synapse.access.http.%s" % (site_tag,), + site_tag, + listener_config, + root_resource, + ), + interface=address + ) + logger.info("Synapse federation_sender now listening on port %d", port) def start_listening(self, listeners): @@ -108,15 +115,22 @@ class FederationSenderServer(HomeServer): if listener["type"] == "http": self._listen_http(listener) elif listener["type"] == "manhole": - reactor.listenTCP( - listener["port"], - manhole( - username="matrix", - password="rabbithole", - globals={"hs": self}, - ), - interface=listener.get("bind_address", '127.0.0.1') - ) + bind_address = listener.get("bind_address", None) + bind_addresses = listener.get("bind_addresses", []) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + listener["port"], + manhole( + username="matrix", + password="rabbithole", + globals={"hs": self}, + ), + interface=address + ) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 54f35900f8..2d6becad1a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -107,7 +107,8 @@ def build_resource_for_web_client(hs): class SynapseHomeServer(HomeServer): def _listener_http(self, config, listener_config): port = listener_config["port"] - bind_address = listener_config.get("bind_address", "") + bind_address = listener_config.get("bind_address", None) + bind_addresses = listener_config.get("bind_addresses", []) tls = listener_config.get("tls", False) site_tag = listener_config.get("tag", port) @@ -173,29 +174,35 @@ class SynapseHomeServer(HomeServer): root_resource = Resource() root_resource = create_resource_tree(resources, root_resource) + + if bind_address: + bind_addresses.append(bind_address) + if tls: - reactor.listenSSL( - port, - SynapseSite( - "synapse.access.https.%s" % (site_tag,), - site_tag, - listener_config, - root_resource, - ), - self.tls_server_context_factory, - interface=bind_address - ) + for address in bind_addresses: + reactor.listenSSL( + port, + SynapseSite( + "synapse.access.https.%s" % (site_tag,), + site_tag, + listener_config, + root_resource, + ), + self.tls_server_context_factory, + interface=address + ) else: - reactor.listenTCP( - port, - SynapseSite( - "synapse.access.http.%s" % (site_tag,), - site_tag, - listener_config, - root_resource, - ), - interface=bind_address - ) + for address in bind_addresses: + reactor.listenTCP( + port, + SynapseSite( + "synapse.access.http.%s" % (site_tag,), + site_tag, + listener_config, + root_resource, + ), + interface=address + ) logger.info("Synapse now listening on port %d", port) def start_listening(self): @@ -205,15 +212,22 @@ class SynapseHomeServer(HomeServer): if listener["type"] == "http": self._listener_http(config, listener) elif listener["type"] == "manhole": - reactor.listenTCP( - listener["port"], - manhole( - username="matrix", - password="rabbithole", - globals={"hs": self}, - ), - interface=listener.get("bind_address", '127.0.0.1') - ) + bind_address = listener.get("bind_address", None) + bind_addresses = listener.get("bind_addresses", []) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + listener["port"], + manhole( + username="matrix", + password="rabbithole", + globals={"hs": self}, + ), + interface=address + ) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 44c19a1bef..c121107245 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -87,7 +87,8 @@ class MediaRepositoryServer(HomeServer): def _listen_http(self, listener_config): port = listener_config["port"] - bind_address = listener_config.get("bind_address", "") + bind_address = listener_config.get("bind_address", None) + bind_addresses = listener_config.get("bind_addresses", []) site_tag = listener_config.get("tag", port) resources = {} for res in listener_config["resources"]: @@ -105,16 +106,22 @@ class MediaRepositoryServer(HomeServer): }) root_resource = create_resource_tree(resources, Resource()) - reactor.listenTCP( - port, - SynapseSite( - "synapse.access.http.%s" % (site_tag,), - site_tag, - listener_config, - root_resource, - ), - interface=bind_address - ) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + port, + SynapseSite( + "synapse.access.http.%s" % (site_tag,), + site_tag, + listener_config, + root_resource, + ), + interface=address + ) + logger.info("Synapse media repository now listening on port %d", port) def start_listening(self, listeners): @@ -122,15 +129,22 @@ class MediaRepositoryServer(HomeServer): if listener["type"] == "http": self._listen_http(listener) elif listener["type"] == "manhole": - reactor.listenTCP( - listener["port"], - manhole( - username="matrix", - password="rabbithole", - globals={"hs": self}, - ), - interface=listener.get("bind_address", '127.0.0.1') - ) + bind_address = listener.get("bind_address", None) + bind_addresses = listener.get("bind_addresses", []) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + listener["port"], + manhole( + username="matrix", + password="rabbithole", + globals={"hs": self}, + ), + interface=address + ) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index a0e765c54f..159850c44c 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -121,7 +121,8 @@ class PusherServer(HomeServer): def _listen_http(self, listener_config): port = listener_config["port"] - bind_address = listener_config.get("bind_address", "") + bind_address = listener_config.get("bind_address", None) + bind_addresses = listener_config.get("bind_addresses", []) site_tag = listener_config.get("tag", port) resources = {} for res in listener_config["resources"]: @@ -130,16 +131,33 @@ class PusherServer(HomeServer): resources[METRICS_PREFIX] = MetricsResource(self) root_resource = create_resource_tree(resources, Resource()) - reactor.listenTCP( - port, - SynapseSite( - "synapse.access.http.%s" % (site_tag,), - site_tag, - listener_config, - root_resource, - ), - interface=bind_address - ) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + port, + SynapseSite( + "synapse.access.http.%s" % (site_tag,), + site_tag, + listener_config, + root_resource, + ), + interface=address + ) + else: + reactor.listenTCP( + port, + SynapseSite( + "synapse.access.http.%s" % (site_tag,), + site_tag, + listener_config, + root_resource, + ), + interface=bind_address + ) + logger.info("Synapse pusher now listening on port %d", port) def start_listening(self, listeners): @@ -147,15 +165,22 @@ class PusherServer(HomeServer): if listener["type"] == "http": self._listen_http(listener) elif listener["type"] == "manhole": - reactor.listenTCP( - listener["port"], - manhole( - username="matrix", - password="rabbithole", - globals={"hs": self}, - ), - interface=listener.get("bind_address", '127.0.0.1') - ) + bind_address = listener.get("bind_address", None) + bind_addresses = listener.get("bind_addresses", []) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + listener["port"], + manhole( + username="matrix", + password="rabbithole", + globals={"hs": self}, + ), + interface=address + ) else: logger.warn("Unrecognized listener type: %s", listener["type"]) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index bf1b995dc2..56143d0025 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -289,7 +289,8 @@ class SynchrotronServer(HomeServer): def _listen_http(self, listener_config): port = listener_config["port"] - bind_address = listener_config.get("bind_address", "") + bind_address = listener_config.get("bind_address", None) + bind_addresses = listener_config.get("bind_addresses", []) site_tag = listener_config.get("tag", port) resources = {} for res in listener_config["resources"]: @@ -310,16 +311,22 @@ class SynchrotronServer(HomeServer): }) root_resource = create_resource_tree(resources, Resource()) - reactor.listenTCP( - port, - SynapseSite( - "synapse.access.http.%s" % (site_tag,), - site_tag, - listener_config, - root_resource, - ), - interface=bind_address - ) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + port, + SynapseSite( + "synapse.access.http.%s" % (site_tag,), + site_tag, + listener_config, + root_resource, + ), + interface=address + ) + logger.info("Synapse synchrotron now listening on port %d", port) def start_listening(self, listeners): @@ -327,15 +334,22 @@ class SynchrotronServer(HomeServer): if listener["type"] == "http": self._listen_http(listener) elif listener["type"] == "manhole": - reactor.listenTCP( - listener["port"], - manhole( - username="matrix", - password="rabbithole", - globals={"hs": self}, - ), - interface=listener.get("bind_address", '127.0.0.1') - ) + bind_address = listener.get("bind_address", None) + bind_addresses = listener.get("bind_addresses", []) + + if bind_address: + bind_addresses.append(bind_address) + + for address in bind_addresses: + reactor.listenTCP( + listener["port"], + manhole( + username="matrix", + password="rabbithole", + globals={"hs": self}, + ), + interface=address + ) else: logger.warn("Unrecognized listener type: %s", listener["type"]) -- cgit 1.4.1 From c95e9fff990722dbeb8bc7971640a517ea7f5fbb Mon Sep 17 00:00:00 2001 From: Johannes Löthberg Date: Sun, 18 Dec 2016 20:54:22 +0100 Subject: Make default homeserver config use bind_addresses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johannes Löthberg --- synapse/config/server.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/config/server.py b/synapse/config/server.py index 634d8e6fe5..1b9e10b527 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -155,9 +155,10 @@ class ServerConfig(Config): # The port to listen for HTTPS requests on. port: %(bind_port)s - # Local interface to listen on. - # The empty string will cause synapse to listen on all interfaces. - bind_address: '' + # Local addresses to listen on. + # This will listen on all IPv4 addresses by default. + bind_addresses: + - '0.0.0.0' # This is a 'http' listener, allows us to specify 'resources'. type: http @@ -188,7 +189,7 @@ class ServerConfig(Config): # For when matrix traffic passes through loadbalancer that unwraps TLS. - port: %(unsecure_port)s tls: false - bind_address: '' + bind_addresses: ['0.0.0.0'] type: http x_forwarded: false -- cgit 1.4.1 From f5cd5ebd7bd8582acd5805021c6718869f8519b1 Mon Sep 17 00:00:00 2001 From: Johannes Löthberg Date: Sun, 18 Dec 2016 23:14:32 +0100 Subject: Add IPv6 comment to default config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johannes Löthberg --- synapse/config/server.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'synapse') diff --git a/synapse/config/server.py b/synapse/config/server.py index 1b9e10b527..5e6b2a68a7 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -159,6 +159,10 @@ class ServerConfig(Config): # This will listen on all IPv4 addresses by default. bind_addresses: - '0.0.0.0' + # Uncomment to listen on all IPv6 interfaces + # N.B: On at least Linux this will also listen on all IPv4 + # addresses, so you will need to comment out the line above. + # - '::' # This is a 'http' listener, allows us to specify 'resources'. type: http -- cgit 1.4.1 From a9c1b419a9c913f8cfb373335c3b7824abcf7406 Mon Sep 17 00:00:00 2001 From: Johannes Löthberg Date: Sun, 18 Dec 2016 23:16:43 +0100 Subject: Bump twisted dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At least 16.0.0 is needed for wrapClientTLS support. Signed-off-by: Johannes Löthberg --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 3742a25b37..7817b0cd91 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -24,7 +24,7 @@ REQUIREMENTS = { "signedjson>=1.0.0": ["signedjson>=1.0.0"], "pynacl==0.3.0": ["nacl==0.3.0", "nacl.bindings"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], - "Twisted>=15.1.0": ["twisted>=15.1.0"], + "Twisted>=16.0.0": ["twisted>=16.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], "pyasn1": ["pyasn1"], -- cgit 1.4.1 From f2a5aebf98a04bc4250ce2800b51d42543b5f35f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 18 Dec 2016 22:25:21 +0000 Subject: fix ability to change password to a non-ascii one https://github.com/vector-im/riot-web/issues/2658 --- synapse/handlers/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 3b146f09d6..652efba455 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -656,8 +656,8 @@ class AuthHandler(BaseHandler): Whether self.hash(password) == stored_hash (bool). """ if stored_hash: - return bcrypt.hashpw(password + self.hs.config.password_pepper, - stored_hash.encode('utf-8')) == stored_hash + return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper, + stored_hash.encode('utf8')) == stored_hash else: return False -- cgit 1.4.1 From 702c020e58dd8663e66c25f88fba2a02264e7357 Mon Sep 17 00:00:00 2001 From: Johannes Löthberg Date: Tue, 20 Dec 2016 01:37:50 +0100 Subject: Fix check for bind_address MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The empty string is a valid setting for the bind_address option, so explicitly check for None here instead. Signed-off-by: Johannes Löthberg --- synapse/app/appservice.py | 4 ++-- synapse/app/client_reader.py | 4 ++-- synapse/app/federation_reader.py | 4 ++-- synapse/app/federation_sender.py | 4 ++-- synapse/app/homeserver.py | 4 ++-- synapse/app/media_repository.py | 4 ++-- synapse/app/pusher.py | 4 ++-- synapse/app/synchrotron.py | 4 ++-- 8 files changed, 16 insertions(+), 16 deletions(-) (limited to 'synapse') diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index e24c1e1eda..c1379fdd7d 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -87,7 +87,7 @@ class AppserviceServer(HomeServer): root_resource = create_resource_tree(resources, Resource()) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: @@ -112,7 +112,7 @@ class AppserviceServer(HomeServer): bind_address = listener.get("bind_address", None) bind_addresses = listener.get("bind_addresses", []) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 305a82b664..b5e1d659e6 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -110,7 +110,7 @@ class ClientReaderServer(HomeServer): root_resource = create_resource_tree(resources, Resource()) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: @@ -135,7 +135,7 @@ class ClientReaderServer(HomeServer): bind_address = listener.get("bind_address", None) bind_addresses = listener.get("bind_addresses", []) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 321dfc7cd5..c6810b83db 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -101,7 +101,7 @@ class FederationReaderServer(HomeServer): root_resource = create_resource_tree(resources, Resource()) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: @@ -126,7 +126,7 @@ class FederationReaderServer(HomeServer): bind_address = listener.get("bind_address", None) bind_addresses = listener.get("bind_addresses", []) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index 8092fd316c..23aae8a09c 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -93,7 +93,7 @@ class FederationSenderServer(HomeServer): root_resource = create_resource_tree(resources, Resource()) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: @@ -118,7 +118,7 @@ class FederationSenderServer(HomeServer): bind_address = listener.get("bind_address", None) bind_addresses = listener.get("bind_addresses", []) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2d6becad1a..6c69ccd7e2 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -175,7 +175,7 @@ class SynapseHomeServer(HomeServer): root_resource = create_resource_tree(resources, root_resource) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) if tls: @@ -215,7 +215,7 @@ class SynapseHomeServer(HomeServer): bind_address = listener.get("bind_address", None) bind_addresses = listener.get("bind_addresses", []) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index c121107245..a47283e520 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -107,7 +107,7 @@ class MediaRepositoryServer(HomeServer): root_resource = create_resource_tree(resources, Resource()) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: @@ -132,7 +132,7 @@ class MediaRepositoryServer(HomeServer): bind_address = listener.get("bind_address", None) bind_addresses = listener.get("bind_addresses", []) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 159850c44c..a3df375c81 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -132,7 +132,7 @@ class PusherServer(HomeServer): root_resource = create_resource_tree(resources, Resource()) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: @@ -168,7 +168,7 @@ class PusherServer(HomeServer): bind_address = listener.get("bind_address", None) bind_addresses = listener.get("bind_addresses", []) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 56143d0025..439daaa60a 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -312,7 +312,7 @@ class SynchrotronServer(HomeServer): root_resource = create_resource_tree(resources, Resource()) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: @@ -337,7 +337,7 @@ class SynchrotronServer(HomeServer): bind_address = listener.get("bind_address", None) bind_addresses = listener.get("bind_addresses", []) - if bind_address: + if bind_address is not None: bind_addresses.append(bind_address) for address in bind_addresses: -- cgit 1.4.1 From 0c88ab184422739a20289ca213861986f70ae6e6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 20 Dec 2016 18:27:30 +0000 Subject: Add /account/3pid/delete endpoint Also fix a typo in a comment --- synapse/handlers/auth.py | 11 ++++++++++ synapse/rest/client/v2_alpha/account.py | 36 ++++++++++++++++++++++++++++++++- synapse/storage/registration.py | 11 ++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 652efba455..ebadace4c1 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -617,6 +617,17 @@ class AuthHandler(BaseHandler): self.hs.get_clock().time_msec() ) + @defer.inlineCallbacks + def delete_threepid(self, user_id, medium, address): + # 'Canonicalise' email addresses as per above + if medium == 'email': + address = address.lower() + + ret = yield self.store.user_delete_threepid( + user_id, medium, address, + ) + defer.returnValue(ret) + def _save_session(self, session): # TODO: Persistent storage logger.debug("Saving session %s", session) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index eb49ad62e9..e74e5e0123 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -241,7 +241,7 @@ class ThreepidRestServlet(RestServlet): for reqd in ['medium', 'address', 'validated_at']: if reqd not in threepid: - logger.warn("Couldn't add 3pid: invalid response from ID sevrer") + logger.warn("Couldn't add 3pid: invalid response from ID server") raise SynapseError(500, "Invalid response from ID Server") yield self.auth_handler.add_threepid( @@ -263,9 +263,43 @@ class ThreepidRestServlet(RestServlet): defer.returnValue((200, {})) +class ThreepidDeleteRestServlet(RestServlet): + PATTERNS = client_v2_patterns("/account/3pid/delete$", releases=()) + + def __init__(self, hs): + super(ThreepidDeleteRestServlet, self).__init__() + self.auth = hs.get_auth() + self.auth_handler = hs.get_auth_handler() + + @defer.inlineCallbacks + def on_POST(self, request): + yield run_on_reactor() + + body = parse_json_object_from_request(request) + + required = ['medium', 'address'] + absent = [] + for k in required: + if k not in body: + absent.append(k) + + if absent: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + + requester = yield self.auth.get_user_by_req(request) + user_id = requester.user.to_string() + + yield self.auth_handler.delete_threepid( + user_id, body['medium'], body['address'] + ) + + defer.returnValue((200, {})) + + def register_servlets(hs, http_server): PasswordRequestTokenRestServlet(hs).register(http_server) PasswordRestServlet(hs).register(http_server) DeactivateAccountRestServlet(hs).register(http_server) ThreepidRequestTokenRestServlet(hs).register(http_server) ThreepidRestServlet(hs).register(http_server) + ThreepidDeleteRestServlet(hs).register(http_server) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 983a8ec52b..26be6060c3 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -413,6 +413,17 @@ class RegistrationStore(background_updates.BackgroundUpdateStore): desc="user_delete_threepids", ) + def user_delete_threepid(self, user_id, medium, address): + return self._simple_delete( + "user_threepids", + keyvalues={ + "user_id": user_id, + "medium": medium, + "address": address, + }, + desc="user_delete_threepids", + ) + @defer.inlineCallbacks def count_all_users(self): """Counts all users registered on the homeserver.""" -- cgit 1.4.1 From 84cf00c6450fb0334e4eac186f05ae6cd6afe2bb Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 21 Dec 2016 09:44:03 +0000 Subject: Fix another comment typo --- synapse/handlers/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index ebadace4c1..221d7ea7a2 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -607,7 +607,7 @@ class AuthHandler(BaseHandler): # types (mediums) of threepid. For now, we still use the existing # infrastructure, but this is the start of synapse gaining knowledge # of specific types of threepid (and fixes the fact that checking - # for the presenc eof an email address during password reset was + # for the presence of an email address during password reset was # case sensitive). if medium == 'email': address = address.lower() -- cgit 1.4.1 From 555d702e3441e355d6b5e23702ab9505728dea71 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 31 Dec 2016 15:21:37 +0000 Subject: limit total timeout for get_missing_events to 10s --- synapse/federation/federation_client.py | 4 +++- synapse/federation/federation_server.py | 5 +++++ synapse/federation/transport/client.py | 5 +++-- 3 files changed, 11 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 6851f2376d..b4bcec77ed 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -707,7 +707,7 @@ class FederationClient(FederationBase): @defer.inlineCallbacks def get_missing_events(self, destination, room_id, earliest_events_ids, - latest_events, limit, min_depth): + latest_events, limit, min_depth, timeout): """Tries to fetch events we are missing. This is called when we receive an event without having received all of its ancestors. @@ -721,6 +721,7 @@ class FederationClient(FederationBase): have all previous events for. limit (int): Maximum number of events to return. min_depth (int): Minimum depth of events tor return. + timeout (int): Max time to wait in ms """ try: content = yield self.transport_layer.get_missing_events( @@ -730,6 +731,7 @@ class FederationClient(FederationBase): latest_events=[e.event_id for e in latest_events], limit=limit, min_depth=min_depth, + timeout=timeout, ) events = [ diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index f4c60e67e3..6d76e6f917 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -425,6 +425,7 @@ class FederationServer(FederationBase): " limit: %d, min_depth: %d", earliest_events, latest_events, limit, min_depth ) + missing_events = yield self.handler.on_get_missing_events( origin, room_id, earliest_events, latest_events, limit, min_depth ) @@ -567,6 +568,9 @@ class FederationServer(FederationBase): len(prevs - seen), pdu.room_id, list(prevs - seen)[:5] ) + # XXX: we set timeout to 10s to help workaround + # https://github.com/matrix-org/synapse/issues/1733 + missing_events = yield self.get_missing_events( origin, pdu.room_id, @@ -574,6 +578,7 @@ class FederationServer(FederationBase): latest_events=[pdu], limit=10, min_depth=min_depth, + timeout=10000, ) # We want to sort these by depth so we process them and diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 491cdc29e1..915af34409 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -386,7 +386,7 @@ class TransportLayerClient(object): @defer.inlineCallbacks @log_function def get_missing_events(self, destination, room_id, earliest_events, - latest_events, limit, min_depth): + latest_events, limit, min_depth, timeout): path = PREFIX + "/get_missing_events/%s" % (room_id,) content = yield self.client.post_json( @@ -397,7 +397,8 @@ class TransportLayerClient(object): "min_depth": int(min_depth), "earliest_events": earliest_events, "latest_events": latest_events, - } + }, + timeout=timeout, ) defer.returnValue(content) -- cgit 1.4.1 From 8e82611f3726bfd577ca77a39328c63ecb29410f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 5 Jan 2017 11:44:44 +0000 Subject: fix comment --- synapse/federation/federation_server.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 6d76e6f917..800f04189f 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -569,7 +569,23 @@ class FederationServer(FederationBase): ) # XXX: we set timeout to 10s to help workaround - # https://github.com/matrix-org/synapse/issues/1733 + # https://github.com/matrix-org/synapse/issues/1733. + # The reason is to avoid holding the linearizer lock + # whilst processing inbound /send transactions, causing + # FDs to stack up and block other inbound transactions + # which empirically can currently take up to 30 minutes. + # + # N.B. this explicitly disables retry attempts. + # + # N.B. this also increases our chances of falling back to + # fetching fresh state for the room if the missing event + # can't be found, which slightly reduces our security. + # it may also increase our DAG extremity count for the room, + # causing additional state resolution? See #1760. + # However, fetching state doesn't hold the linearizer lock + # apparently. + # + # see https://github.com/matrix-org/synapse/pull/1744 missing_events = yield self.get_missing_events( origin, -- cgit 1.4.1 From 2f4b2f4783c608e27dba91e528d2c5a9032b0051 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Jan 2017 04:00:42 +0000 Subject: gah, fix mangled merge of 0.18.7 into develop --- synapse/events/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 8c71aeb5e4..da9f3ad436 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -43,7 +43,7 @@ class _EventInternalMetadata(object): returns a str with the name of the server this event is sent on behalf of. """ - return getattr(self, "get_send_on_behalf_of", None) + return getattr(self, "send_on_behalf_of", None) def _event_dict_property(key): -- cgit 1.4.1 From 9a8ae6f1bf833b58416fae1add1972ac3e9d2d59 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Jan 2017 14:47:56 +0000 Subject: Bump version and changelog --- CHANGES.rst | 11 +++++++++++ synapse/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/CHANGES.rst b/CHANGES.rst index 3fb3197ace..68e9d8c671 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +Changes in synapse v0.18.7 (2017-01-09) +======================================= + +No changes from v0.18.7-rc2 + + Changes in synapse v0.18.7-rc2 (2017-01-07) =========================================== @@ -6,6 +12,7 @@ Bug fixes: * Fix error in rc1's discarding invalid inbound traffic logic that was incorrectly discarding missing events + Changes in synapse v0.18.7-rc1 (2017-01-06) =========================================== @@ -16,6 +23,7 @@ Bug fixes: * Discard inbound federation traffic from invalid domains, to immunise against #1753 + Changes in synapse v0.18.6 (2017-01-06) ======================================= @@ -23,6 +31,7 @@ Bug fixes: * Fix bug when checking if a guest user is allowed to join a room (PR #1772) + Changes in synapse v0.18.6-rc3 (2017-01-05) =========================================== @@ -38,6 +47,7 @@ Changes: * Improve logging for debugging deadlocks (PR #1766, PR #1767) + Changes in synapse v0.18.6-rc2 (2016-12-30) =========================================== @@ -55,6 +65,7 @@ Bug fixes: * Make sure that outbound connections are closed (PR #1725) + Changes in synapse v0.18.5 (2016-12-16) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index cb332279df..498ded38c0 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.18.7-rc2" +__version__ = "0.18.7" -- cgit 1.4.1 From f7085ac84f76ee621ea52c9eaa0399c786d14027 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Jan 2017 17:17:10 +0000 Subject: Name linearizer's for better logs --- synapse/federation/federation_server.py | 4 ++-- synapse/handlers/room_member.py | 2 +- synapse/rest/media/v1/media_repository.py | 2 +- synapse/state.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 1fee4e83a6..862ccbef5d 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -52,8 +52,8 @@ class FederationServer(FederationBase): self.auth = hs.get_auth() - self._room_pdu_linearizer = Linearizer() - self._server_linearizer = Linearizer() + self._room_pdu_linearizer = Linearizer("fed_room_pdu") + self._server_linearizer = Linearizer("fed_server") # We cache responses to state queries, as they take a while and often # come in waves. diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 2f8782e522..649aaf6d29 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -45,7 +45,7 @@ class RoomMemberHandler(BaseHandler): def __init__(self, hs): super(RoomMemberHandler, self).__init__(hs) - self.member_linearizer = Linearizer() + self.member_linearizer = Linearizer(name="member") self.clock = hs.get_clock() diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 692e078419..2b693ae548 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -61,7 +61,7 @@ class MediaRepository(object): self.dynamic_thumbnails = hs.config.dynamic_thumbnails self.thumbnail_requirements = hs.config.thumbnail_requirements - self.remote_media_linearizer = Linearizer() + self.remote_media_linearizer = Linearizer(name="media_remote") self.recently_accessed_remotes = set() diff --git a/synapse/state.py b/synapse/state.py index 8003099c88..b9d5627a82 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -89,7 +89,7 @@ class StateHandler(object): # dict of set of event_ids -> _StateCacheEntry. self._state_cache = None - self.resolve_linearizer = Linearizer() + self.resolve_linearizer = Linearizer(name="state_resolve_lock") def start_caching(self): logger.debug("start_caching") -- cgit 1.4.1 From 6823fe52410db3b95df720b7955ad7b617dc7dee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Jan 2017 18:25:13 +0000 Subject: Linearize updates to membership via PUT /state/ --- synapse/handlers/room_member.py | 19 +++++++++++++++---- synapse/rest/client/v1/room.py | 28 +++++++++++++++++----------- tests/rest/client/v1/test_rooms.py | 4 ++-- tests/rest/client/v1/utils.py | 5 ++++- 4 files changed, 38 insertions(+), 18 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 2f8782e522..8e7bbe9f75 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -89,7 +89,7 @@ class RoomMemberHandler(BaseHandler): duplicate = yield msg_handler.deduplicate_state_event(event, context) if duplicate is not None: # Discard the new event since this membership change is a no-op. - return + defer.returnValue(duplicate) yield msg_handler.handle_new_client_event( requester, @@ -120,6 +120,8 @@ class RoomMemberHandler(BaseHandler): if prev_member_event.membership == Membership.JOIN: user_left_room(self.distributor, target, room_id) + defer.returnValue(event) + @defer.inlineCallbacks def remote_join(self, remote_room_hosts, room_id, user, content): if len(remote_room_hosts) == 0: @@ -187,6 +189,7 @@ class RoomMemberHandler(BaseHandler): ratelimit=True, content=None, ): + content_specified = bool(content) if content is None: content = {} @@ -229,6 +232,12 @@ class RoomMemberHandler(BaseHandler): errcode=Codes.BAD_STATE ) + same_content = content == old_state.content + same_membership = old_membership == effective_membership_state + same_sender = requester.user.to_string() == old_state.sender + if same_sender and same_membership and same_content: + defer.returnValue(old_state) + is_host_in_room = yield self._is_host_in_room(current_state_ids) if effective_membership_state == Membership.JOIN: @@ -247,8 +256,9 @@ class RoomMemberHandler(BaseHandler): content["membership"] = Membership.JOIN profile = self.hs.get_handlers().profile_handler - content["displayname"] = yield profile.get_displayname(target) - content["avatar_url"] = yield profile.get_avatar_url(target) + if not content_specified: + content["displayname"] = yield profile.get_displayname(target) + content["avatar_url"] = yield profile.get_avatar_url(target) if requester.is_guest: content["kind"] = "guest" @@ -290,7 +300,7 @@ class RoomMemberHandler(BaseHandler): defer.returnValue({}) - yield self._local_membership_update( + res = yield self._local_membership_update( requester=requester, target=target, room_id=room_id, @@ -300,6 +310,7 @@ class RoomMemberHandler(BaseHandler): prev_event_ids=latest_event_ids, content=content, ) + defer.returnValue(res) @defer.inlineCallbacks def send_membership_event( diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index eead435bfd..2ebf5e59a0 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -152,23 +152,29 @@ class RoomStateEventRestServlet(ClientV1RestServlet): if state_key is not None: event_dict["state_key"] = state_key - msg_handler = self.handlers.message_handler - event, context = yield msg_handler.create_event( - event_dict, - token_id=requester.access_token_id, - txn_id=txn_id, - ) - if event_type == EventTypes.Member: - yield self.handlers.room_member_handler.send_membership_event( + membership = content.get("membership", None) + event = yield self.handlers.room_member_handler.update_membership( requester, - event, - context, + target=UserID.from_string(state_key), + room_id=room_id, + action=membership, + content=content, ) else: + msg_handler = self.handlers.message_handler + event, context = yield msg_handler.create_event( + event_dict, + token_id=requester.access_token_id, + txn_id=txn_id, + ) + yield msg_handler.send_nonmember_event(requester, event, context) - defer.returnValue((200, {"event_id": event.event_id})) + ret = {} + if event: + ret = {"event_id": event.event_id} + defer.returnValue((200, ret)) # TODO: Needs unit testing for generic events + feedback diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 4fe99ebc0b..6bce352c5f 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -259,8 +259,8 @@ class RoomPermissionsTestCase(RestTestCase): # set [invite/join/left] of self, set [invite/join/left] of other, # expect all 404s because room doesn't exist on any server for usr in [self.user_id, self.rmcreator_id]: - yield self.join(room=room, user=usr, expect_code=403) - yield self.leave(room=room, user=usr, expect_code=403) + yield self.join(room=room, user=usr, expect_code=404) + yield self.leave(room=room, user=usr, expect_code=404) @defer.inlineCallbacks def test_membership_private_room_perms(self): diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index 17524b2e23..3bb1dd003a 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -87,7 +87,10 @@ class RestTestCase(unittest.TestCase): (code, response) = yield self.mock_resource.trigger( "PUT", path, json.dumps(data) ) - self.assertEquals(expect_code, code, msg=str(response)) + self.assertEquals( + expect_code, code, + msg="Expected: %d, got: %d, resp: %r" % (expect_code, code, response) + ) self.auth_user_id = temp_id -- cgit 1.4.1 From 586f474a44cf320c7d578714aea5113f2701073c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Jan 2017 12:46:00 +0000 Subject: Don't block messages sending on bumping presence --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7a57a69bd3..59d0ad3bdc 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -279,7 +279,7 @@ class MessageHandler(BaseHandler): if event.type == EventTypes.Message: presence = self.hs.get_presence_handler() - yield presence.bump_presence_active_time(user) + preserve_fn(presence.bump_presence_active_time)(user) @defer.inlineCallbacks def deduplicate_state_event(self, event, context): -- cgit 1.4.1 From f477370c0cb5b3f6bee74eb2f8209f7fdf5db9fe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Jan 2017 14:04:13 +0000 Subject: Add paranoia exception catch in Linearizer --- synapse/util/async.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/util/async.py b/synapse/util/async.py index 83875edc85..35380bf8ed 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -192,8 +192,11 @@ class Linearizer(object): logger.info( "Waiting to acquire linearizer lock %r for key %r", self.name, key ) - with PreserveLoggingContext(): - yield current_defer + try: + with PreserveLoggingContext(): + yield current_defer + except: + logger.exception("Unexpected exception in Linearizer") logger.info("Acquired linearizer lock %r for key %r", self.name, key) -- cgit 1.4.1 From 3b4de17d2be01a0726152f614eb03078979ec631 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Jan 2017 14:05:53 +0000 Subject: Comment --- synapse/handlers/message.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 59d0ad3bdc..88bd2d572e 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -279,6 +279,8 @@ class MessageHandler(BaseHandler): if event.type == EventTypes.Message: presence = self.hs.get_presence_handler() + # We don't want to block sending messages on any presence code. This + # matters as sometimes presence code can take a while. preserve_fn(presence.bump_presence_active_time)(user) @defer.inlineCallbacks -- cgit 1.4.1 From 32019c98971fae775fe79bb30615899e7f6b09d4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Jan 2017 14:19:50 +0000 Subject: Log which files we saved attachments to in the media_repository --- synapse/rest/media/v1/media_repository.py | 4 ++++ synapse/rest/media/v1/thumbnailer.py | 5 +++++ synapse/rest/media/v1/upload_resource.py | 2 ++ 3 files changed, 11 insertions(+) (limited to 'synapse') diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 2b693ae548..3cbeca503c 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -98,6 +98,8 @@ class MediaRepository(object): with open(fname, "wb") as f: f.write(content) + logger.info("Stored local media in file %r", fname) + yield self.store.store_local_media( media_id=media_id, media_type=media_type, @@ -190,6 +192,8 @@ class MediaRepository(object): else: upload_name = None + logger.info("Stored remote media in file %r", fname) + yield self.store.store_cached_remote_media( origin=server_name, media_id=media_id, diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py index 0bb3676844..3868d4f65f 100644 --- a/synapse/rest/media/v1/thumbnailer.py +++ b/synapse/rest/media/v1/thumbnailer.py @@ -16,6 +16,10 @@ import PIL.Image as Image from io import BytesIO +import logging + +logger = logging.getLogger(__name__) + class Thumbnailer(object): @@ -86,4 +90,5 @@ class Thumbnailer(object): output_bytes = output_bytes_io.getvalue() with open(output_path, "wb") as output_file: output_file.write(output_bytes) + logger.info("Stored thumbnail in file %r", output_path) return len(output_bytes) diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index b716d1d892..4ab33f73bf 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -97,6 +97,8 @@ class UploadResource(Resource): content_length, requester.user ) + logger.info("Uploaded content with URI %r", content_uri) + respond_with_json( request, 200, {"content_uri": content_uri}, send_cors=True ) -- cgit 1.4.1 From dd52d4de4c9318b377a880b2662124688fc23129 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Jan 2017 14:34:50 +0000 Subject: Limit number of entries to prefill from cache Some tables, like device_inbox, take a long time to query at startup for the stream change cache prefills. This is likely because they are slower growing streams and so are more fragmented on disk. For now, lets pull fewer entries out to make startup quicker. In future, we should add a better index to make it even faster. --- synapse/storage/__init__.py | 4 +++- synapse/storage/_base.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index fe936b3e62..e8495f1eb9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -189,7 +189,8 @@ class DataStore(RoomMemberStore, RoomStore, db_conn, "device_inbox", entity_column="user_id", stream_column="stream_id", - max_value=max_device_inbox_id + max_value=max_device_inbox_id, + limit=1000, ) self._device_inbox_stream_cache = StreamChangeCache( "DeviceInboxStreamChangeCache", min_device_inbox_id, @@ -202,6 +203,7 @@ class DataStore(RoomMemberStore, RoomStore, entity_column="destination", stream_column="stream_id", max_value=max_device_inbox_id, + limit=1000, ) self._device_federation_outbox_stream_cache = StreamChangeCache( "DeviceFederationOutboxStreamChangeCache", min_device_outbox_id, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index b62c459d8b..5620a655eb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -838,18 +838,19 @@ class SQLBaseStore(object): return txn.execute(sql, keyvalues.values()) def _get_cache_dict(self, db_conn, table, entity_column, stream_column, - max_value): + max_value, limit=100000): # Fetch a mapping of room_id -> max stream position for "recent" rooms. # It doesn't really matter how many we get, the StreamChangeCache will # do the right thing to ensure it respects the max size of cache. sql = ( "SELECT %(entity)s, MAX(%(stream)s) FROM %(table)s" - " WHERE %(stream)s > ? - 100000" + " WHERE %(stream)s > ? - %(limit)s" " GROUP BY %(entity)s" ) % { "table": table, "entity": entity_column, "stream": stream_column, + "limit": limit, } sql = self.database_engine.convert_param_style(sql) -- cgit 1.4.1