From 16e0680498435d2a93b26f81f57bfa41c52c691b Mon Sep 17 00:00:00 2001 From: Benoît S Date: Thu, 21 Feb 2019 18:44:10 +0100 Subject: Added HAProxy example (#4660) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added HAProxy example Proposal of an example with HAProxy. Asked by #4541. Signed-off-by: Benoît S. (“Benpro”) * Following suggestions of @richvdh --- docs/reverse_proxy.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'docs') diff --git a/docs/reverse_proxy.rst b/docs/reverse_proxy.rst index d8aaac8a08..242935a62f 100644 --- a/docs/reverse_proxy.rst +++ b/docs/reverse_proxy.rst @@ -85,6 +85,24 @@ Let's assume that we expect clients to connect to our server at +* HAProxy:: + + frontend https + bind 0.0.0.0:443 v4v6 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1 + bind :::443 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1 + + # Matrix client traffic + acl matrix hdr(host) -i matrix.example.com + use_backend matrix if matrix + + frontend matrix-federation + bind 0.0.0.0:8448 v4v6 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1 + bind :::8448 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1 + default_backend matrix + + backend matrix + server matrix 127.0.0.1:8008 + You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true`` for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are recorded correctly. -- cgit 1.5.1 From fcd6f01dc790c1245b413e37d405a8d092397f98 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 22 Feb 2019 10:56:42 +0000 Subject: Minor tweaks to acme docs (#4689) --- changelog.d/4689.misc | 1 + docs/ACME.md | 19 +++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 changelog.d/4689.misc (limited to 'docs') diff --git a/changelog.d/4689.misc b/changelog.d/4689.misc new file mode 100644 index 0000000000..15c4d9404b --- /dev/null +++ b/changelog.d/4689.misc @@ -0,0 +1 @@ +Minor tweaks to acme docs. diff --git a/docs/ACME.md b/docs/ACME.md index e555c7c939..46136a9f2c 100644 --- a/docs/ACME.md +++ b/docs/ACME.md @@ -10,13 +10,14 @@ through [Let's Encrypt](https://letsencrypt.org/) if you tell it to. In the case that your `server_name` config variable is the same as the hostname that the client connects to, then the same certificate can be -used between client and federation ports without issue. +used between client and federation ports without issue. -For a sample configuration, please inspect the new ACME section in the example -generated config by running the `generate-config` executable. For example: +If your configuration file does not already have an `acme` section, you can +generate an example config by running the `generate_config` executable. For +example: ``` -~/synapse/env3/bin/generate-config +~/synapse/env3/bin/generate_config ``` You will need to provide Let's Encrypt (or another ACME provider) access to @@ -27,10 +28,9 @@ like `authbind` to allow Synapse to listen on port 80 without root access. (Do not run Synapse with root permissions!) Detailed instructions are available under "ACME setup" below. -If you are already using self-signed certificates, you will need to back up -or delete them (files `example.com.tls.crt` and `example.com.tls.key` in -Synapse's root directory), Synapse's ACME implementation will not overwrite -them. +If you already have certificates, you will need to back up or delete them +(files `example.com.tls.crt` and `example.com.tls.key` in Synapse's root +directory), Synapse's ACME implementation will not overwrite them. You may wish to use alternate methods such as Certbot to obtain a certificate from Let's Encrypt, depending on your server configuration. Of course, if you @@ -87,7 +87,6 @@ acme: port: 8009 ``` - #### Authbind `authbind` allows a program which does not run as root to bind to @@ -127,4 +126,4 @@ acme: Ensure that the certificate paths specified in `homeserver.yaml` (`tls_certificate_path` and `tls_private_key_path`) do not currently point to any files. Synapse will not provision certificates if files exist, as it does not want to overwrite existing certificates. -Finally, start/restart Synapse. \ No newline at end of file +Finally, start/restart Synapse. -- cgit 1.5.1 From 313987187ee04dce5e70db17c1ab9377f283be7e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Feb 2019 15:04:34 +0000 Subject: Fix tightloop over connecting to replication server If the client failed to process incoming commands during the initial set up of the replication connection it would immediately disconnect and reconnect, resulting in a tightloop. This can happen, for example, when subscribing to a stream that has a row that is too long in the backlog. The fix here is to not consider the connection successfully set up until the client has succesfully subscribed and caught up with the streams. This ensures that the retry logic timers aren't reset until then, meaning that if an error does happen during start up the client will continue backing off before retrying again. --- docs/tcp_replication.rst | 4 +++- synapse/replication/tcp/client.py | 38 ++++++++++++++++++++++++++++++++++--- synapse/replication/tcp/commands.py | 5 ++++- 3 files changed, 42 insertions(+), 5 deletions(-) (limited to 'docs') diff --git a/docs/tcp_replication.rst b/docs/tcp_replication.rst index 73436cea62..75e723484c 100644 --- a/docs/tcp_replication.rst +++ b/docs/tcp_replication.rst @@ -188,7 +188,9 @@ RDATA (S) A single update in a stream POSITION (S) - The position of the stream has been updated + The position of the stream has been updated. Sent to the client after all + missing updates for a stream have been sent to the client and they're now + up to date. ERROR (S, C) There was an error diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 586dddb40b..914cd24b55 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -54,7 +54,6 @@ class ReplicationClientFactory(ReconnectingClientFactory): def buildProtocol(self, addr): logger.info("Connected to replication: %r", addr) - self.resetDelay() return ClientReplicationStreamProtocol( self.client_name, self.server_name, self._clock, self.handler ) @@ -90,15 +89,23 @@ class ReplicationClientHandler(object): # Used for tests. self.awaiting_syncs = {} + # Set of stream names that have been subscribe to, but haven't yet + # caught up with. This is used to track when the client has been fully + # connected to the remote. + self.streams_connecting = None + + # The factory used to create connections. + self.factory = None + def start_replication(self, hs): """Helper method to start a replication connection to the remote server using TCP. """ client_name = hs.config.worker_name - factory = ReplicationClientFactory(hs, client_name, self) + self.factory = ReplicationClientFactory(hs, client_name, self) host = hs.config.worker_replication_host port = hs.config.worker_replication_port - hs.get_reactor().connectTCP(host, port, factory) + hs.get_reactor().connectTCP(host, port, self.factory) def on_rdata(self, stream_name, token, rows): """Called when we get new replication data. By default this just pokes @@ -115,6 +122,12 @@ class ReplicationClientHandler(object): Can be overriden in subclasses to handle more. """ + # When we get a `POSITION` command it means we've finished getting + # missing updates for the given stream, and are now up to date. + self.streams_connecting.discard(stream_name) + if not self.streams_connecting: + self.finished_connecting() + return self.store.process_replication_rows(stream_name, token, []) def on_sync(self, data): @@ -140,6 +153,10 @@ class ReplicationClientHandler(object): args["account_data"] = user_account_data elif room_account_data: args["account_data"] = room_account_data + + # Record which streams we're in the process of subscribing to + self.streams_connecting = set(args.keys()) + return args def get_currently_syncing_users(self): @@ -204,3 +221,18 @@ class ReplicationClientHandler(object): for cmd in self.pending_commands: connection.send_command(cmd) self.pending_commands = [] + + # This will happen if we don't actually subscribe to any streams + if not self.streams_connecting: + self.finished_connecting() + + def finished_connecting(self): + """Called when we have successfully subscribed and caught up to all + streams we're interested in. + """ + logger.info("Finished connecting to server") + + # We don't reset the delay any earlier as otherwise if there is a + # problem during start up we'll end up tight looping connecting to the + # server. + self.factory.resetDelay() diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py index 327556f6a1..2098c32a77 100644 --- a/synapse/replication/tcp/commands.py +++ b/synapse/replication/tcp/commands.py @@ -127,8 +127,11 @@ class RdataCommand(Command): class PositionCommand(Command): - """Sent by the client to tell the client the stream postition without + """Sent by the server to tell the client the stream postition without needing to send an RDATA. + + Sent to the client after all missing updates for a stream have been sent + to the client and they're now up to date. """ NAME = "POSITION" -- cgit 1.5.1 From 4bc74835185a42fae0004c30e55f61f5bb8c78d2 Mon Sep 17 00:00:00 2001 From: Paul Tötterman Date: Tue, 26 Feb 2019 20:01:45 +0200 Subject: Fix apache reverse proxy example (#4742) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So that it actually works. See https://httpd.apache.org/docs/2.4/mod/mod_proxy.html#proxypass Signed-off-by: Paul Tötterman --- docs/reverse_proxy.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docs') diff --git a/docs/reverse_proxy.rst b/docs/reverse_proxy.rst index 242935a62f..4706061eba 100644 --- a/docs/reverse_proxy.rst +++ b/docs/reverse_proxy.rst @@ -79,7 +79,7 @@ Let's assume that we expect clients to connect to our server at SSLEngine on ServerName example.com; - + ProxyPass http://127.0.0.1:8008/_matrix nocanon ProxyPassReverse http://127.0.0.1:8008/_matrix -- cgit 1.5.1 From 4cff9376f7b07d3c60b1f882cde5848149862dd9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Feb 2019 13:43:53 +0000 Subject: Move server key queries to federation reader --- docs/workers.rst | 1 + synapse/app/federation_reader.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'docs') diff --git a/docs/workers.rst b/docs/workers.rst index 3ba5879f76..7552455a4e 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -182,6 +182,7 @@ endpoints matching the following regular expressions:: ^/_matrix/federation/v1/event_auth/ ^/_matrix/federation/v1/exchange_third_party_invite/ ^/_matrix/federation/v1/send/ + ^/_matrix/key/v2/query The above endpoints should all be routed to the federation_reader worker by the reverse-proxy configuration. diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index b116c17669..7da79dc827 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -21,7 +21,7 @@ from twisted.web.resource import NoResource import synapse from synapse import events -from synapse.api.urls import FEDERATION_PREFIX +from synapse.api.urls import FEDERATION_PREFIX, SERVER_KEY_V2_PREFIX from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig @@ -44,6 +44,7 @@ from synapse.replication.slave.storage.registration import SlavedRegistrationSto from synapse.replication.slave.storage.room import RoomStore from synapse.replication.slave.storage.transactions import SlavedTransactionStore from synapse.replication.tcp.client import ReplicationClientHandler +from synapse.rest.key.v2 import KeyApiV2Resource from synapse.server import HomeServer from synapse.storage.engines import create_engine from synapse.util.httpresourcetree import create_resource_tree @@ -99,6 +100,9 @@ class FederationReaderServer(HomeServer): ), }) + if name in ["keys", "federation"]: + resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( -- cgit 1.5.1 From 54f9ce11a7100e7207bb068a84dfaba886995c2c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Feb 2019 14:26:08 +0000 Subject: Move /account/3pid to client_reader --- docs/workers.rst | 1 + synapse/app/client_reader.py | 2 ++ synapse/storage/registration.py | 66 ++++++++++++++++++++--------------------- 3 files changed, 36 insertions(+), 33 deletions(-) (limited to 'docs') diff --git a/docs/workers.rst b/docs/workers.rst index 3ba5879f76..0170fba898 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -223,6 +223,7 @@ following regular expressions:: ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$ ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$ ^/_matrix/client/(api/v1|r0|unstable)/login$ + ^/_matrix/client/(api/v1|r0|unstable)/account/3pid$ Additionally, the following REST endpoints can be handled, but all requests must be routed to the same instance:: diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 043b48f8f3..5070094cad 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -48,6 +48,7 @@ from synapse.rest.client.v1.room import ( RoomMemberListRestServlet, RoomStateRestServlet, ) +from synapse.rest.client.v2_alpha.account import ThreepidRestServlet from synapse.rest.client.v2_alpha.register import RegisterRestServlet from synapse.server import HomeServer from synapse.storage.engines import create_engine @@ -96,6 +97,7 @@ class ClientReaderServer(HomeServer): RoomEventContextServlet(self).register(resource) RegisterRestServlet(self).register(resource) LoginRestServlet(self).register(resource) + ThreepidRestServlet(self).register(resource) resources.update({ "/_matrix/client/r0": resource, diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 9b9572890b..9b6c28892c 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -295,6 +295,39 @@ class RegistrationWorkerStore(SQLBaseStore): return ret['user_id'] return None + @defer.inlineCallbacks + def user_add_threepid(self, user_id, medium, address, validated_at, added_at): + yield self._simple_upsert("user_threepids", { + "medium": medium, + "address": address, + }, { + "user_id": user_id, + "validated_at": validated_at, + "added_at": added_at, + }) + + @defer.inlineCallbacks + def user_get_threepids(self, user_id): + ret = yield self._simple_select_list( + "user_threepids", { + "user_id": user_id + }, + ['medium', 'address', 'validated_at', 'added_at'], + 'user_get_threepids' + ) + defer.returnValue(ret) + + def user_delete_threepid(self, user_id, medium, address): + return self._simple_delete( + "user_threepids", + keyvalues={ + "user_id": user_id, + "medium": medium, + "address": address, + }, + desc="user_delete_threepids", + ) + class RegistrationStore(RegistrationWorkerStore, background_updates.BackgroundUpdateStore): @@ -632,39 +665,6 @@ class RegistrationStore(RegistrationWorkerStore, defer.returnValue(res if res else False) - @defer.inlineCallbacks - def user_add_threepid(self, user_id, medium, address, validated_at, added_at): - yield self._simple_upsert("user_threepids", { - "medium": medium, - "address": address, - }, { - "user_id": user_id, - "validated_at": validated_at, - "added_at": added_at, - }) - - @defer.inlineCallbacks - def user_get_threepids(self, user_id): - ret = yield self._simple_select_list( - "user_threepids", { - "user_id": user_id - }, - ['medium', 'address', 'validated_at', 'added_at'], - 'user_get_threepids' - ) - defer.returnValue(ret) - - def user_delete_threepid(self, user_id, medium, address): - return self._simple_delete( - "user_threepids", - keyvalues={ - "user_id": user_id, - "medium": medium, - "address": address, - }, - desc="user_delete_threepids", - ) - @defer.inlineCallbacks def save_or_get_3pid_guest_access_token( self, medium, address, access_token, inviter_user_id -- cgit 1.5.1