From d19bccdbecfeee3e59666748db7fd971cd7978d2 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 13 May 2021 14:37:20 -0400 Subject: Update SSO mapping providers documentation about unique IDs. (#9980) --- docs/sso_mapping_providers.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'docs') diff --git a/docs/sso_mapping_providers.md b/docs/sso_mapping_providers.md index 50020d1a4a..6db2dc8be5 100644 --- a/docs/sso_mapping_providers.md +++ b/docs/sso_mapping_providers.md @@ -67,8 +67,8 @@ A custom mapping provider must specify the following methods: - Arguments: - `userinfo` - A `authlib.oidc.core.claims.UserInfo` object to extract user information from. - - This method must return a string, which is the unique identifier for the - user. Commonly the ``sub`` claim of the response. + - This method must return a string, which is the unique, immutable identifier + for the user. Commonly the `sub` claim of the response. * `map_user_attributes(self, userinfo, token, failures)` - This method must be async. - Arguments: @@ -87,7 +87,9 @@ A custom mapping provider must specify the following methods: `localpart` value, such as `john.doe1`. - Returns a dictionary with two keys: - `localpart`: A string, used to generate the Matrix ID. If this is - `None`, the user is prompted to pick their own username. + `None`, the user is prompted to pick their own username. This is only used + during a user's first login. Once a localpart has been associated with a + remote user ID (see `get_remote_user_id`) it cannot be updated. - `displayname`: An optional string, the display name for the user. * `get_extra_attributes(self, userinfo, token)` - This method must be async. @@ -153,8 +155,8 @@ A custom mapping provider must specify the following methods: information from. - `client_redirect_url` - A string, the URL that the client will be redirected to. - - This method must return a string, which is the unique identifier for the - user. Commonly the ``uid`` claim of the response. + - This method must return a string, which is the unique, immutable identifier + for the user. Commonly the `uid` claim of the response. * `saml_response_to_user_attributes(self, saml_response, failures, client_redirect_url)` - Arguments: - `saml_response` - A `saml2.response.AuthnResponse` object to extract user @@ -172,8 +174,10 @@ A custom mapping provider must specify the following methods: redirected to. - This method must return a dictionary, which will then be used by Synapse to build a new user. The following keys are allowed: - * `mxid_localpart` - The mxid localpart of the new user. If this is - `None`, the user is prompted to pick their own username. + * `mxid_localpart` - A string, the mxid localpart of the new user. If this is + `None`, the user is prompted to pick their own username. This is only used + during a user's first login. Once a localpart has been associated with a + remote user ID (see `get_remote_user_id`) it cannot be updated. * `displayname` - The displayname of the new user. If not provided, will default to the value of `mxid_localpart`. * `emails` - A list of emails for the new user. If not provided, will -- cgit 1.5.1 From 976216959b3a216a3403b953338f92852c45645c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 14 May 2021 09:21:00 +0100 Subject: Update minimum supported version in postgres.md (#9988) --- changelog.d/9988.doc | 1 + docs/postgres.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/9988.doc (limited to 'docs') diff --git a/changelog.d/9988.doc b/changelog.d/9988.doc new file mode 100644 index 0000000000..87cda376a6 --- /dev/null +++ b/changelog.d/9988.doc @@ -0,0 +1 @@ +Fix outdated minimum PostgreSQL version in postgres.md. diff --git a/docs/postgres.md b/docs/postgres.md index 680685d04e..b99fad8a6e 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -1,6 +1,6 @@ # Using Postgres -Postgres version 9.5 or later is known to work. +Synapse supports PostgreSQL versions 9.6 or later. ## Install postgres client libraries -- cgit 1.5.1 From c14f99be461d8ac9a36ad548e8e463feeda6394c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 14 May 2021 10:51:08 +0100 Subject: Support enabling opentracing by user (#9978) Add a config option which allows enabling opentracing by user id, eg for debugging requests made by a test user. --- changelog.d/9978.feature | 1 + docs/opentracing.md | 10 +++++----- docs/sample_config.yaml | 20 ++++++++++++++------ synapse/api/auth.py | 5 +++++ synapse/config/tracer.py | 37 +++++++++++++++++++++++++++++++------ 5 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 changelog.d/9978.feature (limited to 'docs') diff --git a/changelog.d/9978.feature b/changelog.d/9978.feature new file mode 100644 index 0000000000..851adb9f6e --- /dev/null +++ b/changelog.d/9978.feature @@ -0,0 +1 @@ +Add a configuration option which allows enabling opentracing by user id. diff --git a/docs/opentracing.md b/docs/opentracing.md index 4c7a56a5d7..f91362f112 100644 --- a/docs/opentracing.md +++ b/docs/opentracing.md @@ -42,17 +42,17 @@ To receive OpenTracing spans, start up a Jaeger server. This can be done using docker like so: ```sh -docker run -d --name jaeger +docker run -d --name jaeger \ -p 6831:6831/udp \ -p 6832:6832/udp \ -p 5778:5778 \ -p 16686:16686 \ -p 14268:14268 \ - jaegertracing/all-in-one:1.13 + jaegertracing/all-in-one:1 ``` Latest documentation is probably at - +https://www.jaegertracing.io/docs/latest/getting-started. ## Enable OpenTracing in Synapse @@ -62,7 +62,7 @@ as shown in the [sample config](./sample_config.yaml). For example: ```yaml opentracing: - tracer_enabled: true + enabled: true homeserver_whitelist: - "mytrustedhomeserver.org" - "*.myotherhomeservers.com" @@ -90,4 +90,4 @@ to two problems, namely: ## Configuring Jaeger Sampling strategies can be set as in this document: - +. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 67ad57b1aa..2952f2ba32 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -2845,7 +2845,8 @@ opentracing: #enabled: true # The list of homeservers we wish to send and receive span contexts and span baggage. - # See docs/opentracing.rst + # See docs/opentracing.rst. + # # This is a list of regexes which are matched against the server_name of the # homeserver. # @@ -2854,19 +2855,26 @@ opentracing: #homeserver_whitelist: # - ".*" + # A list of the matrix IDs of users whose requests will always be traced, + # even if the tracing system would otherwise drop the traces due to + # probabilistic sampling. + # + # By default, the list is empty. + # + #force_tracing_for_users: + # - "@user1:server_name" + # - "@user2:server_name" + # Jaeger can be configured to sample traces at different rates. # All configuration options provided by Jaeger can be set here. - # Jaeger's configuration mostly related to trace sampling which + # Jaeger's configuration is mostly related to trace sampling which # is documented here: - # https://www.jaegertracing.io/docs/1.13/sampling/. + # https://www.jaegertracing.io/docs/latest/sampling/. # #jaeger_config: # sampler: # type: const # param: 1 - - # Logging whether spans were started and reported - # # logging: # false diff --git a/synapse/api/auth.py b/synapse/api/auth.py index efc926d094..458306eba5 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -87,6 +87,7 @@ class Auth: ) self._track_appservice_user_ips = hs.config.track_appservice_user_ips self._macaroon_secret_key = hs.config.macaroon_secret_key + self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users async def check_from_context( self, room_version: str, event, context, do_sig_check=True @@ -208,6 +209,8 @@ class Auth: opentracing.set_tag("authenticated_entity", user_id) opentracing.set_tag("user_id", user_id) opentracing.set_tag("appservice_id", app_service.id) + if user_id in self._force_tracing_for_users: + opentracing.set_tag(opentracing.tags.SAMPLING_PRIORITY, 1) return requester @@ -260,6 +263,8 @@ class Auth: opentracing.set_tag("user_id", user_info.user_id) if device_id: opentracing.set_tag("device_id", device_id) + if user_info.token_owner in self._force_tracing_for_users: + opentracing.set_tag(opentracing.tags.SAMPLING_PRIORITY, 1) return requester except KeyError: diff --git a/synapse/config/tracer.py b/synapse/config/tracer.py index db22b5b19f..d0ea17261f 100644 --- a/synapse/config/tracer.py +++ b/synapse/config/tracer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Set + from synapse.python_dependencies import DependencyException, check_requirements from ._base import Config, ConfigError @@ -32,6 +34,8 @@ class TracerConfig(Config): {"sampler": {"type": "const", "param": 1}, "logging": False}, ) + self.force_tracing_for_users: Set[str] = set() + if not self.opentracer_enabled: return @@ -48,6 +52,19 @@ class TracerConfig(Config): if not isinstance(self.opentracer_whitelist, list): raise ConfigError("Tracer homeserver_whitelist config is malformed") + force_tracing_for_users = opentracing_config.get("force_tracing_for_users", []) + if not isinstance(force_tracing_for_users, list): + raise ConfigError( + "Expected a list", ("opentracing", "force_tracing_for_users") + ) + for i, u in enumerate(force_tracing_for_users): + if not isinstance(u, str): + raise ConfigError( + "Expected a string", + ("opentracing", "force_tracing_for_users", f"index {i}"), + ) + self.force_tracing_for_users.add(u) + def generate_config_section(cls, **kwargs): return """\ ## Opentracing ## @@ -64,7 +81,8 @@ class TracerConfig(Config): #enabled: true # The list of homeservers we wish to send and receive span contexts and span baggage. - # See docs/opentracing.rst + # See docs/opentracing.rst. + # # This is a list of regexes which are matched against the server_name of the # homeserver. # @@ -73,19 +91,26 @@ class TracerConfig(Config): #homeserver_whitelist: # - ".*" + # A list of the matrix IDs of users whose requests will always be traced, + # even if the tracing system would otherwise drop the traces due to + # probabilistic sampling. + # + # By default, the list is empty. + # + #force_tracing_for_users: + # - "@user1:server_name" + # - "@user2:server_name" + # Jaeger can be configured to sample traces at different rates. # All configuration options provided by Jaeger can be set here. - # Jaeger's configuration mostly related to trace sampling which + # Jaeger's configuration is mostly related to trace sampling which # is documented here: - # https://www.jaegertracing.io/docs/1.13/sampling/. + # https://www.jaegertracing.io/docs/latest/sampling/. # #jaeger_config: # sampler: # type: const # param: 1 - - # Logging whether spans were started and reported - # # logging: # false """ -- cgit 1.5.1 From 66609122260ad151359b9c0028634094cf51b5c5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 14 May 2021 13:14:48 +0100 Subject: Update postgres docs (#9989) --- changelog.d/9988.doc | 2 +- changelog.d/9989.doc | 1 + docs/postgres.md | 198 +++++++++++++++++++++++++-------------------------- 3 files changed, 98 insertions(+), 103 deletions(-) create mode 100644 changelog.d/9989.doc (limited to 'docs') diff --git a/changelog.d/9988.doc b/changelog.d/9988.doc index 87cda376a6..25338c44c3 100644 --- a/changelog.d/9988.doc +++ b/changelog.d/9988.doc @@ -1 +1 @@ -Fix outdated minimum PostgreSQL version in postgres.md. +Updates to the PostgreSQL documentation (`postgres.md`). diff --git a/changelog.d/9989.doc b/changelog.d/9989.doc new file mode 100644 index 0000000000..25338c44c3 --- /dev/null +++ b/changelog.d/9989.doc @@ -0,0 +1 @@ +Updates to the PostgreSQL documentation (`postgres.md`). diff --git a/docs/postgres.md b/docs/postgres.md index b99fad8a6e..f83155e52a 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -33,28 +33,15 @@ Assuming your PostgreSQL database user is called `postgres`, first authenticate # Or, if your system uses sudo to get administrative rights sudo -u postgres bash -Then, create a user ``synapse_user`` with: +Then, create a postgres user and a database with: + # this will prompt for a password for the new user createuser --pwprompt synapse_user -Before you can authenticate with the `synapse_user`, you must create a -database that it can access. To create a database, first connect to the -database with your database user: + createdb --encoding=UTF8 --locale=C --template=template0 --owner=synapse_user synapse - su - postgres # Or: sudo -u postgres bash - psql - -and then run: - - CREATE DATABASE synapse - ENCODING 'UTF8' - LC_COLLATE='C' - LC_CTYPE='C' - template=template0 - OWNER synapse_user; - -This would create an appropriate database named `synapse` owned by the -`synapse_user` user (which must already have been created as above). +The above will create a user called `synapse_user`, and a database called +`synapse`. Note that the PostgreSQL database *must* have the correct encoding set (as shown above), otherwise it will not be able to store UTF8 strings. @@ -63,79 +50,6 @@ You may need to enable password authentication so `synapse_user` can connect to the database. See . -If you get an error along the lines of `FATAL: Ident authentication failed for -user "synapse_user"`, you may need to use an authentication method other than -`ident`: - -* If the `synapse_user` user has a password, add the password to the `database:` - section of `homeserver.yaml`. Then add the following to `pg_hba.conf`: - - ``` - host synapse synapse_user ::1/128 md5 # or `scram-sha-256` instead of `md5` if you use that - ``` - -* If the `synapse_user` user does not have a password, then a password doesn't - have to be added to `homeserver.yaml`. But the following does need to be added - to `pg_hba.conf`: - - ``` - host synapse synapse_user ::1/128 trust - ``` - -Note that line order matters in `pg_hba.conf`, so make sure that if you do add a -new line, it is inserted before: - -``` -host all all ::1/128 ident -``` - -### Fixing incorrect `COLLATE` or `CTYPE` - -Synapse will refuse to set up a new database if it has the wrong values of -`COLLATE` and `CTYPE` set, and will log warnings on existing databases. Using -different locales can cause issues if the locale library is updated from -underneath the database, or if a different version of the locale is used on any -replicas. - -The safest way to fix the issue is to take a dump and recreate the database with -the correct `COLLATE` and `CTYPE` parameters (as shown above). It is also possible to change the -parameters on a live database and run a `REINDEX` on the entire database, -however extreme care must be taken to avoid database corruption. - -Note that the above may fail with an error about duplicate rows if corruption -has already occurred, and such duplicate rows will need to be manually removed. - - -## Fixing inconsistent sequences error - -Synapse uses Postgres sequences to generate IDs for various tables. A sequence -and associated table can get out of sync if, for example, Synapse has been -downgraded and then upgraded again. - -To fix the issue shut down Synapse (including any and all workers) and run the -SQL command included in the error message. Once done Synapse should start -successfully. - - -## Tuning Postgres - -The default settings should be fine for most deployments. For larger -scale deployments tuning some of the settings is recommended, details of -which can be found at -. - -In particular, we've found tuning the following values helpful for -performance: - -- `shared_buffers` -- `effective_cache_size` -- `work_mem` -- `maintenance_work_mem` -- `autovacuum_work_mem` - -Note that the appropriate values for those fields depend on the amount -of free memory the database host has available. - ## Synapse config When you are ready to start using PostgreSQL, edit the `database` @@ -165,18 +79,42 @@ may block for an extended period while it waits for a response from the database server. Example values might be: ```yaml -# seconds of inactivity after which TCP should send a keepalive message to the server -keepalives_idle: 10 +database: + args: + # ... as above + + # seconds of inactivity after which TCP should send a keepalive message to the server + keepalives_idle: 10 -# the number of seconds after which a TCP keepalive message that is not -# acknowledged by the server should be retransmitted -keepalives_interval: 10 + # the number of seconds after which a TCP keepalive message that is not + # acknowledged by the server should be retransmitted + keepalives_interval: 10 -# the number of TCP keepalives that can be lost before the client's connection -# to the server is considered dead -keepalives_count: 3 + # the number of TCP keepalives that can be lost before the client's connection + # to the server is considered dead + keepalives_count: 3 ``` +## Tuning Postgres + +The default settings should be fine for most deployments. For larger +scale deployments tuning some of the settings is recommended, details of +which can be found at +. + +In particular, we've found tuning the following values helpful for +performance: + +- `shared_buffers` +- `effective_cache_size` +- `work_mem` +- `maintenance_work_mem` +- `autovacuum_work_mem` + +Note that the appropriate values for those fields depend on the amount +of free memory the database host has available. + + ## Porting from SQLite ### Overview @@ -185,9 +123,8 @@ The script `synapse_port_db` allows porting an existing synapse server backed by SQLite to using PostgreSQL. This is done in as a two phase process: -1. Copy the existing SQLite database to a separate location (while the - server is down) and running the port script against that offline - database. +1. Copy the existing SQLite database to a separate location and run + the port script against that offline database. 2. Shut down the server. Rerun the port script to port any data that has come in since taking the first snapshot. Restart server against the PostgreSQL database. @@ -245,3 +182,60 @@ PostgreSQL database configuration file `homeserver-postgres.yaml`: ./synctl start Synapse should now be running against PostgreSQL. + + +## Troubleshooting + +### Alternative auth methods + +If you get an error along the lines of `FATAL: Ident authentication failed for +user "synapse_user"`, you may need to use an authentication method other than +`ident`: + +* If the `synapse_user` user has a password, add the password to the `database:` + section of `homeserver.yaml`. Then add the following to `pg_hba.conf`: + + ``` + host synapse synapse_user ::1/128 md5 # or `scram-sha-256` instead of `md5` if you use that + ``` + +* If the `synapse_user` user does not have a password, then a password doesn't + have to be added to `homeserver.yaml`. But the following does need to be added + to `pg_hba.conf`: + + ``` + host synapse synapse_user ::1/128 trust + ``` + +Note that line order matters in `pg_hba.conf`, so make sure that if you do add a +new line, it is inserted before: + +``` +host all all ::1/128 ident +``` + +### Fixing incorrect `COLLATE` or `CTYPE` + +Synapse will refuse to set up a new database if it has the wrong values of +`COLLATE` and `CTYPE` set, and will log warnings on existing databases. Using +different locales can cause issues if the locale library is updated from +underneath the database, or if a different version of the locale is used on any +replicas. + +The safest way to fix the issue is to dump the database and recreate it with +the correct locale parameter (as shown above). It is also possible to change the +parameters on a live database and run a `REINDEX` on the entire database, +however extreme care must be taken to avoid database corruption. + +Note that the above may fail with an error about duplicate rows if corruption +has already occurred, and such duplicate rows will need to be manually removed. + +### Fixing inconsistent sequences error + +Synapse uses Postgres sequences to generate IDs for various tables. A sequence +and associated table can get out of sync if, for example, Synapse has been +downgraded and then upgraded again. + +To fix the issue shut down Synapse (including any and all workers) and run the +SQL command included in the error message. Once done Synapse should start +successfully. -- cgit 1.5.1 From 4d6e5a5e995590efe44855d10dcd2a89b841dae8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 18 May 2021 14:13:45 +0100 Subject: Use a database table to hold the users that should have full presence sent to them, instead of something in-memory (#9823) --- changelog.d/9823.misc | 1 + docs/presence_router_module.md | 6 +- synapse/handlers/presence.py | 136 +++++++-- synapse/module_api/__init__.py | 63 ++--- synapse/replication/http/presence.py | 11 +- synapse/rest/admin/server_notice_servlet.py | 8 +- synapse/storage/databases/main/presence.py | 58 +++- .../delta/59/13users_to_send_full_presence_to.sql | 34 +++ tests/events/test_presence_router.py | 15 +- tests/module_api/test_api.py | 303 +++++++++++++++------ tests/replication/test_sharded_event_persister.py | 2 +- 11 files changed, 479 insertions(+), 158 deletions(-) create mode 100644 changelog.d/9823.misc create mode 100644 synapse/storage/schema/main/delta/59/13users_to_send_full_presence_to.sql (limited to 'docs') diff --git a/changelog.d/9823.misc b/changelog.d/9823.misc new file mode 100644 index 0000000000..bf924ab68c --- /dev/null +++ b/changelog.d/9823.misc @@ -0,0 +1 @@ +Allow sending full presence to users via workers other than the one that called `ModuleApi.send_local_online_presence_to`. \ No newline at end of file diff --git a/docs/presence_router_module.md b/docs/presence_router_module.md index d6566d978d..d2844915df 100644 --- a/docs/presence_router_module.md +++ b/docs/presence_router_module.md @@ -28,7 +28,11 @@ async def ModuleApi.send_local_online_presence_to(users: Iterable[str]) -> None which can be given a list of local or remote MXIDs to broadcast known, online user presence to (for those users that the receiving user is considered interested in). It does not include state for users who are currently offline, and it can only be -called on workers that support sending federation. +called on workers that support sending federation. Additionally, this method must +only be called from the process that has been configured to write to the +the [presence stream](https://github.com/matrix-org/synapse/blob/master/docs/workers.md#stream-writers). +By default, this is the main process, but another worker can be configured to do +so. ### Module structure diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6fd1f34289..f5a049d754 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -222,9 +222,21 @@ class BasePresenceHandler(abc.ABC): @abc.abstractmethod async def set_state( - self, target_user: UserID, state: JsonDict, ignore_status_msg: bool = False + self, + target_user: UserID, + state: JsonDict, + ignore_status_msg: bool = False, + force_notify: bool = False, ) -> None: - """Set the presence state of the user. """ + """Set the presence state of the user. + + Args: + target_user: The ID of the user to set the presence state of. + state: The presence state as a JSON dictionary. + ignore_status_msg: True to ignore the "status_msg" field of the `state` dict. + If False, the user's current status will be updated. + force_notify: Whether to force notification of the update to clients. + """ @abc.abstractmethod async def bump_presence_active_time(self, user: UserID): @@ -296,6 +308,51 @@ class BasePresenceHandler(abc.ABC): for destinations, states in hosts_and_states: self._federation.send_presence_to_destinations(states, destinations) + async def send_full_presence_to_users(self, user_ids: Collection[str]): + """ + Adds to the list of users who should receive a full snapshot of presence + upon their next sync. Note that this only works for local users. + + Then, grabs the current presence state for a given set of users and adds it + to the top of the presence stream. + + Args: + user_ids: The IDs of the local users to send full presence to. + """ + # Retrieve one of the users from the given set + if not user_ids: + raise Exception( + "send_full_presence_to_users must be called with at least one user" + ) + user_id = next(iter(user_ids)) + + # Mark all users as receiving full presence on their next sync + await self.store.add_users_to_send_full_presence_to(user_ids) + + # Add a new entry to the presence stream. Since we use stream tokens to determine whether a + # local user should receive a full snapshot of presence when they sync, we need to bump the + # presence stream so that subsequent syncs with no presence activity in between won't result + # in the client receiving multiple full snapshots of presence. + # + # If we bump the stream ID, then the user will get a higher stream token next sync, and thus + # correctly won't receive a second snapshot. + + # Get the current presence state for one of the users (defaults to offline if not found) + current_presence_state = await self.get_state(UserID.from_string(user_id)) + + # Convert the UserPresenceState object into a serializable dict + state = { + "presence": current_presence_state.state, + "status_message": current_presence_state.status_msg, + } + + # Copy the presence state to the tip of the presence stream. + + # We set force_notify=True here so that this presence update is guaranteed to + # increment the presence stream ID (which resending the current user's presence + # otherwise would not do). + await self.set_state(UserID.from_string(user_id), state, force_notify=True) + class _NullContextManager(ContextManager[None]): """A context manager which does nothing.""" @@ -480,8 +537,17 @@ class WorkerPresenceHandler(BasePresenceHandler): target_user: UserID, state: JsonDict, ignore_status_msg: bool = False, + force_notify: bool = False, ) -> None: - """Set the presence state of the user.""" + """Set the presence state of the user. + + Args: + target_user: The ID of the user to set the presence state of. + state: The presence state as a JSON dictionary. + ignore_status_msg: True to ignore the "status_msg" field of the `state` dict. + If False, the user's current status will be updated. + force_notify: Whether to force notification of the update to clients. + """ presence = state["presence"] valid_presence = ( @@ -508,6 +574,7 @@ class WorkerPresenceHandler(BasePresenceHandler): user_id=user_id, state=state, ignore_status_msg=ignore_status_msg, + force_notify=force_notify, ) async def bump_presence_active_time(self, user: UserID) -> None: @@ -677,13 +744,19 @@ class PresenceHandler(BasePresenceHandler): [self.user_to_current_state[user_id] for user_id in unpersisted] ) - async def _update_states(self, new_states: Iterable[UserPresenceState]) -> None: + async def _update_states( + self, new_states: Iterable[UserPresenceState], force_notify: bool = False + ) -> None: """Updates presence of users. Sets the appropriate timeouts. Pokes the notifier and federation if and only if the changed presence state should be sent to clients/servers. Args: new_states: The new user presence state updates to process. + force_notify: Whether to force notifying clients of this presence state update, + even if it doesn't change the state of a user's presence (e.g online -> online). + This is currently used to bump the max presence stream ID without changing any + user's presence (see PresenceHandler.add_users_to_send_full_presence_to). """ now = self.clock.time_msec() @@ -720,6 +793,9 @@ class PresenceHandler(BasePresenceHandler): now=now, ) + if force_notify: + should_notify = True + self.user_to_current_state[user_id] = new_state if should_notify: @@ -1058,9 +1134,21 @@ class PresenceHandler(BasePresenceHandler): await self._update_states(updates) async def set_state( - self, target_user: UserID, state: JsonDict, ignore_status_msg: bool = False + self, + target_user: UserID, + state: JsonDict, + ignore_status_msg: bool = False, + force_notify: bool = False, ) -> None: - """Set the presence state of the user.""" + """Set the presence state of the user. + + Args: + target_user: The ID of the user to set the presence state of. + state: The presence state as a JSON dictionary. + ignore_status_msg: True to ignore the "status_msg" field of the `state` dict. + If False, the user's current status will be updated. + force_notify: Whether to force notification of the update to clients. + """ status_msg = state.get("status_msg", None) presence = state["presence"] @@ -1091,7 +1179,9 @@ class PresenceHandler(BasePresenceHandler): ): new_fields["last_active_ts"] = self.clock.time_msec() - await self._update_states([prev_state.copy_and_replace(**new_fields)]) + await self._update_states( + [prev_state.copy_and_replace(**new_fields)], force_notify=force_notify + ) async def is_visible(self, observed_user: UserID, observer_user: UserID) -> bool: """Returns whether a user can see another user's presence.""" @@ -1389,11 +1479,10 @@ class PresenceEventSource: # # Presence -> Notifier -> PresenceEventSource -> Presence # - # Same with get_module_api, get_presence_router + # Same with get_presence_router: # # AuthHandler -> Notifier -> PresenceEventSource -> ModuleApi -> AuthHandler self.get_presence_handler = hs.get_presence_handler - self.get_module_api = hs.get_module_api self.get_presence_router = hs.get_presence_router self.clock = hs.get_clock() self.store = hs.get_datastore() @@ -1424,16 +1513,21 @@ class PresenceEventSource: stream_change_cache = self.store.presence_stream_cache with Measure(self.clock, "presence.get_new_events"): - if user_id in self.get_module_api()._send_full_presence_to_local_users: - # This user has been specified by a module to receive all current, online - # user presence. Removing from_key and setting include_offline to false - # will do effectively this. - from_key = None - include_offline = False - if from_key is not None: from_key = int(from_key) + # Check if this user should receive all current, online user presence. We only + # bother to do this if from_key is set, as otherwise the user will receive all + # user presence anyways. + if await self.store.should_user_receive_full_presence_with_token( + user_id, from_key + ): + # This user has been specified by a module to receive all current, online + # user presence. Removing from_key and setting include_offline to false + # will do effectively this. + from_key = None + include_offline = False + max_token = self.store.get_current_presence_token() if from_key == max_token: # This is necessary as due to the way stream ID generators work @@ -1467,12 +1561,6 @@ class PresenceEventSource: user_id, include_offline, from_key ) - # Remove the user from the list of users to receive all presence - if user_id in self.get_module_api()._send_full_presence_to_local_users: - self.get_module_api()._send_full_presence_to_local_users.remove( - user_id - ) - return presence_updates, max_token # Make mypy happy. users_interested_in should now be a set @@ -1522,10 +1610,6 @@ class PresenceEventSource: ) presence_updates = list(users_to_state.values()) - # Remove the user from the list of users to receive all presence - if user_id in self.get_module_api()._send_full_presence_to_local_users: - self.get_module_api()._send_full_presence_to_local_users.remove(user_id) - if not include_offline: # Filter out offline presence states presence_updates = self._filter_offline_presence_state(presence_updates) diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index a1a2b9aecc..cecdc96bf5 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -56,14 +56,6 @@ class ModuleApi: self._http_client = hs.get_simple_http_client() # type: SimpleHttpClient self._public_room_list_manager = PublicRoomListManager(hs) - # The next time these users sync, they will receive the current presence - # state of all local users. Users are added by send_local_online_presence_to, - # and removed after a successful sync. - # - # We make this a private variable to deter modules from accessing it directly, - # though other classes in Synapse will still do so. - self._send_full_presence_to_local_users = set() - @property def http_client(self): """Allows making outbound HTTP requests to remote resources. @@ -405,39 +397,44 @@ class ModuleApi: Updates to remote users will be sent immediately, whereas local users will receive them on their next sync attempt. - Note that this method can only be run on the main or federation_sender worker - processes. + Note that this method can only be run on the process that is configured to write to the + presence stream. By default this is the main process. """ - if not self._hs.should_send_federation(): + if self._hs._instance_name not in self._hs.config.worker.writers.presence: raise Exception( "send_local_online_presence_to can only be run " - "on processes that send federation", + "on the process that is configured to write to the " + "presence stream (by default this is the main process)", ) + local_users = set() + remote_users = set() for user in users: if self._hs.is_mine_id(user): - # Modify SyncHandler._generate_sync_entry_for_presence to call - # presence_source.get_new_events with an empty `from_key` if - # that user's ID were in a list modified by ModuleApi somewhere. - # That user would then get all presence state on next incremental sync. - - # Force a presence initial_sync for this user next time - self._send_full_presence_to_local_users.add(user) + local_users.add(user) else: - # Retrieve presence state for currently online users that this user - # is considered interested in - presence_events, _ = await self._presence_stream.get_new_events( - UserID.from_string(user), from_key=None, include_offline=False - ) - - # Send to remote destinations. - - # We pull out the presence handler here to break a cyclic - # dependency between the presence router and module API. - presence_handler = self._hs.get_presence_handler() - await presence_handler.maybe_send_presence_to_interested_destinations( - presence_events - ) + remote_users.add(user) + + # We pull out the presence handler here to break a cyclic + # dependency between the presence router and module API. + presence_handler = self._hs.get_presence_handler() + + if local_users: + # Force a presence initial_sync for these users next time they sync. + await presence_handler.send_full_presence_to_users(local_users) + + for user in remote_users: + # Retrieve presence state for currently online users that this user + # is considered interested in. + presence_events, _ = await self._presence_stream.get_new_events( + UserID.from_string(user), from_key=None, include_offline=False + ) + + # Send to remote destinations. + destination = UserID.from_string(user).domain + presence_handler.get_federation_queue().send_presence_to_destinations( + presence_events, destination + ) class PublicRoomListManager: diff --git a/synapse/replication/http/presence.py b/synapse/replication/http/presence.py index f25307620d..bb00247953 100644 --- a/synapse/replication/http/presence.py +++ b/synapse/replication/http/presence.py @@ -73,6 +73,7 @@ class ReplicationPresenceSetState(ReplicationEndpoint): { "state": { ... }, "ignore_status_msg": false, + "force_notify": false } 200 OK @@ -91,17 +92,23 @@ class ReplicationPresenceSetState(ReplicationEndpoint): self._presence_handler = hs.get_presence_handler() @staticmethod - async def _serialize_payload(user_id, state, ignore_status_msg=False): + async def _serialize_payload( + user_id, state, ignore_status_msg=False, force_notify=False + ): return { "state": state, "ignore_status_msg": ignore_status_msg, + "force_notify": force_notify, } async def _handle_request(self, request, user_id): content = parse_json_object_from_request(request) await self._presence_handler.set_state( - UserID.from_string(user_id), content["state"], content["ignore_status_msg"] + UserID.from_string(user_id), + content["state"], + content["ignore_status_msg"], + content["force_notify"], ) return ( diff --git a/synapse/rest/admin/server_notice_servlet.py b/synapse/rest/admin/server_notice_servlet.py index cc3ab5854b..b5e4c474ef 100644 --- a/synapse/rest/admin/server_notice_servlet.py +++ b/synapse/rest/admin/server_notice_servlet.py @@ -54,7 +54,6 @@ class SendServerNoticeServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() self.txns = HttpTransactionCache(hs) - self.snm = hs.get_server_notices_manager() def register(self, json_resource: HttpServer): PATTERN = "/send_server_notice" @@ -77,7 +76,10 @@ class SendServerNoticeServlet(RestServlet): event_type = body.get("type", EventTypes.Message) state_key = body.get("state_key") - if not self.snm.is_enabled(): + # We grab the server notices manager here as its initialisation has a check for worker processes, + # but worker processes still need to initialise SendServerNoticeServlet (as it is part of the + # admin api). + if not self.hs.get_server_notices_manager().is_enabled(): raise SynapseError(400, "Server notices are not enabled on this server") user_id = body["user_id"] @@ -85,7 +87,7 @@ class SendServerNoticeServlet(RestServlet): if not self.hs.is_mine_id(user_id): raise SynapseError(400, "Server notices can only be sent to local users") - event = await self.snm.send_notice( + event = await self.hs.get_server_notices_manager().send_notice( user_id=body["user_id"], type=event_type, state_key=state_key, diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py index db22fab23e..669a2af884 100644 --- a/synapse/storage/databases/main/presence.py +++ b/synapse/storage/databases/main/presence.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import TYPE_CHECKING, Dict, List, Tuple +from typing import TYPE_CHECKING, Dict, Iterable, List, Tuple from synapse.api.presence import PresenceState, UserPresenceState from synapse.replication.tcp.streams import PresenceStream @@ -57,6 +57,7 @@ class PresenceStore(SQLBaseStore): db_conn, "presence_stream", "stream_id" ) + self.hs = hs self._presence_on_startup = self._get_active_presence(db_conn) presence_cache_prefill, min_presence_val = self.db_pool.get_cache_dict( @@ -210,6 +211,61 @@ class PresenceStore(SQLBaseStore): return {row["user_id"]: UserPresenceState(**row) for row in rows} + async def should_user_receive_full_presence_with_token( + self, + user_id: str, + from_token: int, + ) -> bool: + """Check whether the given user should receive full presence using the stream token + they're updating from. + + Args: + user_id: The ID of the user to check. + from_token: The stream token included in their /sync token. + + Returns: + True if the user should have full presence sent to them, False otherwise. + """ + + def _should_user_receive_full_presence_with_token_txn(txn): + sql = """ + SELECT 1 FROM users_to_send_full_presence_to + WHERE user_id = ? + AND presence_stream_id >= ? + """ + txn.execute(sql, (user_id, from_token)) + return bool(txn.fetchone()) + + return await self.db_pool.runInteraction( + "should_user_receive_full_presence_with_token", + _should_user_receive_full_presence_with_token_txn, + ) + + async def add_users_to_send_full_presence_to(self, user_ids: Iterable[str]): + """Adds to the list of users who should receive a full snapshot of presence + upon their next sync. + + Args: + user_ids: An iterable of user IDs. + """ + # Add user entries to the table, updating the presence_stream_id column if the user already + # exists in the table. + await self.db_pool.simple_upsert_many( + table="users_to_send_full_presence_to", + key_names=("user_id",), + key_values=[(user_id,) for user_id in user_ids], + value_names=("presence_stream_id",), + # We save the current presence stream ID token along with the user ID entry so + # that when a user /sync's, even if they syncing multiple times across separate + # devices at different times, each device will receive full presence once - when + # the presence stream ID in their sync token is less than the one in the table + # for their user ID. + value_values=( + (self._presence_id_gen.get_current_token(),) for _ in user_ids + ), + desc="add_users_to_send_full_presence_to", + ) + async def get_presence_for_all_users( self, include_offline: bool = True, diff --git a/synapse/storage/schema/main/delta/59/13users_to_send_full_presence_to.sql b/synapse/storage/schema/main/delta/59/13users_to_send_full_presence_to.sql new file mode 100644 index 0000000000..07b0f53ecf --- /dev/null +++ b/synapse/storage/schema/main/delta/59/13users_to_send_full_presence_to.sql @@ -0,0 +1,34 @@ +/* Copyright 2021 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Add a table that keeps track of a list of users who should, upon their next +-- sync request, receive presence for all currently online users that they are +-- "interested" in. + +-- The motivation for a DB table over an in-memory list is so that this list +-- can be added to and retrieved from by any worker. Specifically, we don't +-- want to duplicate work across multiple sync workers. + +CREATE TABLE IF NOT EXISTS users_to_send_full_presence_to( + -- The user ID to send full presence to. + user_id TEXT PRIMARY KEY, + -- A presence stream ID token - the current presence stream token when the row was last upserted. + -- If a user calls /sync and this token is part of the update they're to receive, we also include + -- full user presence in the response. + -- This allows multiple devices for a user to receive full presence whenever they next call /sync. + presence_stream_id BIGINT, + FOREIGN KEY (user_id) + REFERENCES users (name) +); \ No newline at end of file diff --git a/tests/events/test_presence_router.py b/tests/events/test_presence_router.py index 01d257307c..875b0d0a11 100644 --- a/tests/events/test_presence_router.py +++ b/tests/events/test_presence_router.py @@ -302,11 +302,18 @@ class PresenceRouterTestCase(FederatingHomeserverTestCase): ) # Check that the expected presence updates were sent - expected_users = [ + # We explicitly compare using sets as we expect that calling + # module_api.send_local_online_presence_to will create a presence + # update that is a duplicate of the specified user's current presence. + # These are sent to clients and will be picked up below, thus we use a + # set to deduplicate. We're just interested that non-offline updates were + # sent out for each user ID. + expected_users = { self.other_user_id, self.presence_receiving_user_one_id, self.presence_receiving_user_two_id, - ] + } + found_users = set() calls = ( self.hs.get_federation_transport_client().send_transaction.call_args_list @@ -326,12 +333,12 @@ class PresenceRouterTestCase(FederatingHomeserverTestCase): # EDUs can contain multiple presence updates for presence_update in edu["content"]["push"]: # Check for presence updates that contain the user IDs we're after - expected_users.remove(presence_update["user_id"]) + found_users.add(presence_update["user_id"]) # Ensure that no offline states are being sent out self.assertNotEqual(presence_update["presence"], "offline") - self.assertEqual(len(expected_users), 0) + self.assertEqual(found_users, expected_users) def send_presence_update( diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index 742ad14b8c..2c68b9a13c 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -13,6 +13,8 @@ # limitations under the License. from unittest.mock import Mock +from twisted.internet import defer + from synapse.api.constants import EduTypes from synapse.events import EventBase from synapse.federation.units import Transaction @@ -22,11 +24,13 @@ from synapse.rest.client.v1 import login, presence, room from synapse.types import create_requester from tests.events.test_presence_router import send_presence_update, sync_presence +from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.test_utils.event_injection import inject_member_event -from tests.unittest import FederatingHomeserverTestCase, override_config +from tests.unittest import HomeserverTestCase, override_config +from tests.utils import USE_POSTGRES_FOR_TESTS -class ModuleApiTestCase(FederatingHomeserverTestCase): +class ModuleApiTestCase(HomeserverTestCase): servlets = [ admin.register_servlets, login.register_servlets, @@ -217,97 +221,16 @@ class ModuleApiTestCase(FederatingHomeserverTestCase): ) self.assertFalse(is_in_public_rooms) - # The ability to send federation is required by send_local_online_presence_to. - @override_config({"send_federation": True}) def test_send_local_online_presence_to(self): - """Tests that send_local_presence_to_users sends local online presence to local users.""" - # Create a user who will send presence updates - self.presence_receiver_id = self.register_user("presence_receiver", "monkey") - self.presence_receiver_tok = self.login("presence_receiver", "monkey") - - # And another user that will send presence updates out - self.presence_sender_id = self.register_user("presence_sender", "monkey") - self.presence_sender_tok = self.login("presence_sender", "monkey") - - # Put them in a room together so they will receive each other's presence updates - room_id = self.helper.create_room_as( - self.presence_receiver_id, - tok=self.presence_receiver_tok, - ) - self.helper.join(room_id, self.presence_sender_id, tok=self.presence_sender_tok) - - # Presence sender comes online - send_presence_update( - self, - self.presence_sender_id, - self.presence_sender_tok, - "online", - "I'm online!", - ) - - # Presence receiver should have received it - presence_updates, sync_token = sync_presence(self, self.presence_receiver_id) - self.assertEqual(len(presence_updates), 1) - - presence_update = presence_updates[0] # type: UserPresenceState - self.assertEqual(presence_update.user_id, self.presence_sender_id) - self.assertEqual(presence_update.state, "online") - - # Syncing again should result in no presence updates - presence_updates, sync_token = sync_presence( - self, self.presence_receiver_id, sync_token - ) - self.assertEqual(len(presence_updates), 0) - - # Trigger sending local online presence - self.get_success( - self.module_api.send_local_online_presence_to( - [ - self.presence_receiver_id, - ] - ) - ) - - # Presence receiver should have received online presence again - presence_updates, sync_token = sync_presence( - self, self.presence_receiver_id, sync_token - ) - self.assertEqual(len(presence_updates), 1) - - presence_update = presence_updates[0] # type: UserPresenceState - self.assertEqual(presence_update.user_id, self.presence_sender_id) - self.assertEqual(presence_update.state, "online") - - # Presence sender goes offline - send_presence_update( - self, - self.presence_sender_id, - self.presence_sender_tok, - "offline", - "I slink back into the darkness.", - ) - - # Trigger sending local online presence - self.get_success( - self.module_api.send_local_online_presence_to( - [ - self.presence_receiver_id, - ] - ) - ) - - # Presence receiver should *not* have received offline state - presence_updates, sync_token = sync_presence( - self, self.presence_receiver_id, sync_token - ) - self.assertEqual(len(presence_updates), 0) + # Test sending local online presence to users from the main process + _test_sending_local_online_presence_to_local_user(self, test_with_workers=False) @override_config({"send_federation": True}) def test_send_local_online_presence_to_federation(self): """Tests that send_local_presence_to_users sends local online presence to remote users.""" # Create a user who will send presence updates - self.presence_sender_id = self.register_user("presence_sender", "monkey") - self.presence_sender_tok = self.login("presence_sender", "monkey") + self.presence_sender_id = self.register_user("presence_sender1", "monkey") + self.presence_sender_tok = self.login("presence_sender1", "monkey") # And a room they're a part of room_id = self.helper.create_room_as( @@ -374,3 +297,209 @@ class ModuleApiTestCase(FederatingHomeserverTestCase): found_update = True self.assertTrue(found_update) + + +class ModuleApiWorkerTestCase(BaseMultiWorkerStreamTestCase): + """For testing ModuleApi functionality in a multi-worker setup""" + + # Testing stream ID replication from the main to worker processes requires postgres + # (due to needing `MultiWriterIdGenerator`). + if not USE_POSTGRES_FOR_TESTS: + skip = "Requires Postgres" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + presence.register_servlets, + ] + + def default_config(self): + conf = super().default_config() + conf["redis"] = {"enabled": "true"} + conf["stream_writers"] = {"presence": ["presence_writer"]} + conf["instance_map"] = { + "presence_writer": {"host": "testserv", "port": 1001}, + } + return conf + + def prepare(self, reactor, clock, homeserver): + self.module_api = homeserver.get_module_api() + self.sync_handler = homeserver.get_sync_handler() + + def test_send_local_online_presence_to_workers(self): + # Test sending local online presence to users from a worker process + _test_sending_local_online_presence_to_local_user(self, test_with_workers=True) + + +def _test_sending_local_online_presence_to_local_user( + test_case: HomeserverTestCase, test_with_workers: bool = False +): + """Tests that send_local_presence_to_users sends local online presence to local users. + + This simultaneously tests two different usecases: + * Testing that this method works when either called from a worker or the main process. + - We test this by calling this method from both a TestCase that runs in monolith mode, and one that + runs with a main and generic_worker. + * Testing that multiple devices syncing simultaneously will all receive a snapshot of local, + online presence - but only once per device. + + Args: + test_with_workers: If True, this method will call ModuleApi.send_local_online_presence_to on a + worker process. The test users will still sync with the main process. The purpose of testing + with a worker is to check whether a Synapse module running on a worker can inform other workers/ + the main process that they should include additional presence when a user next syncs. + """ + if test_with_workers: + # Create a worker process to make module_api calls against + worker_hs = test_case.make_worker_hs( + "synapse.app.generic_worker", {"worker_name": "presence_writer"} + ) + + # Create a user who will send presence updates + test_case.presence_receiver_id = test_case.register_user( + "presence_receiver1", "monkey" + ) + test_case.presence_receiver_tok = test_case.login("presence_receiver1", "monkey") + + # And another user that will send presence updates out + test_case.presence_sender_id = test_case.register_user("presence_sender2", "monkey") + test_case.presence_sender_tok = test_case.login("presence_sender2", "monkey") + + # Put them in a room together so they will receive each other's presence updates + room_id = test_case.helper.create_room_as( + test_case.presence_receiver_id, + tok=test_case.presence_receiver_tok, + ) + test_case.helper.join( + room_id, test_case.presence_sender_id, tok=test_case.presence_sender_tok + ) + + # Presence sender comes online + send_presence_update( + test_case, + test_case.presence_sender_id, + test_case.presence_sender_tok, + "online", + "I'm online!", + ) + + # Presence receiver should have received it + presence_updates, sync_token = sync_presence( + test_case, test_case.presence_receiver_id + ) + test_case.assertEqual(len(presence_updates), 1) + + presence_update = presence_updates[0] # type: UserPresenceState + test_case.assertEqual(presence_update.user_id, test_case.presence_sender_id) + test_case.assertEqual(presence_update.state, "online") + + if test_with_workers: + # Replicate the current sync presence token from the main process to the worker process. + # We need to do this so that the worker process knows the current presence stream ID to + # insert into the database when we call ModuleApi.send_local_online_presence_to. + test_case.replicate() + + # Syncing again should result in no presence updates + presence_updates, sync_token = sync_presence( + test_case, test_case.presence_receiver_id, sync_token + ) + test_case.assertEqual(len(presence_updates), 0) + + # We do an (initial) sync with a second "device" now, getting a new sync token. + # We'll use this in a moment. + _, sync_token_second_device = sync_presence( + test_case, test_case.presence_receiver_id + ) + + # Determine on which process (main or worker) to call ModuleApi.send_local_online_presence_to on + if test_with_workers: + module_api_to_use = worker_hs.get_module_api() + else: + module_api_to_use = test_case.module_api + + # Trigger sending local online presence. We expect this information + # to be saved to the database where all processes can access it. + # Note that we're syncing via the master. + d = module_api_to_use.send_local_online_presence_to( + [ + test_case.presence_receiver_id, + ] + ) + d = defer.ensureDeferred(d) + + if test_with_workers: + # In order for the required presence_set_state replication request to occur between the + # worker and main process, we need to pump the reactor. Otherwise, the coordinator that + # reads the request on the main process won't do so, and the request will time out. + while not d.called: + test_case.reactor.advance(0.1) + + test_case.get_success(d) + + # The presence receiver should have received online presence again. + presence_updates, sync_token = sync_presence( + test_case, test_case.presence_receiver_id, sync_token + ) + test_case.assertEqual(len(presence_updates), 1) + + presence_update = presence_updates[0] # type: UserPresenceState + test_case.assertEqual(presence_update.user_id, test_case.presence_sender_id) + test_case.assertEqual(presence_update.state, "online") + + # We attempt to sync with the second sync token we received above - just to check that + # multiple syncing devices will each receive the necessary online presence. + presence_updates, sync_token_second_device = sync_presence( + test_case, test_case.presence_receiver_id, sync_token_second_device + ) + test_case.assertEqual(len(presence_updates), 1) + + presence_update = presence_updates[0] # type: UserPresenceState + test_case.assertEqual(presence_update.user_id, test_case.presence_sender_id) + test_case.assertEqual(presence_update.state, "online") + + # However, if we now sync with either "device", we won't receive another burst of online presence + # until the API is called again sometime in the future + presence_updates, sync_token = sync_presence( + test_case, test_case.presence_receiver_id, sync_token + ) + + # Now we check that we don't receive *offline* updates using ModuleApi.send_local_online_presence_to. + + # Presence sender goes offline + send_presence_update( + test_case, + test_case.presence_sender_id, + test_case.presence_sender_tok, + "offline", + "I slink back into the darkness.", + ) + + # Presence receiver should have received the updated, offline state + presence_updates, sync_token = sync_presence( + test_case, test_case.presence_receiver_id, sync_token + ) + test_case.assertEqual(len(presence_updates), 1) + + # Now trigger sending local online presence. + d = module_api_to_use.send_local_online_presence_to( + [ + test_case.presence_receiver_id, + ] + ) + d = defer.ensureDeferred(d) + + if test_with_workers: + # In order for the required presence_set_state replication request to occur between the + # worker and main process, we need to pump the reactor. Otherwise, the coordinator that + # reads the request on the main process won't do so, and the request will time out. + while not d.called: + test_case.reactor.advance(0.1) + + test_case.get_success(d) + + # Presence receiver should *not* have received offline state + presence_updates, sync_token = sync_presence( + test_case, test_case.presence_receiver_id, sync_token + ) + test_case.assertEqual(len(presence_updates), 0) diff --git a/tests/replication/test_sharded_event_persister.py b/tests/replication/test_sharded_event_persister.py index d739eb6b17..5eca5c165d 100644 --- a/tests/replication/test_sharded_event_persister.py +++ b/tests/replication/test_sharded_event_persister.py @@ -30,7 +30,7 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): """Checks event persisting sharding works""" # Event persister sharding requires postgres (due to needing - # `MutliWriterIdGenerator`). + # `MultiWriterIdGenerator`). if not USE_POSTGRES_FOR_TESTS: skip = "Requires Postgres" -- cgit 1.5.1 From 5bba1b49058a648197f217268a3978d8acf09c51 Mon Sep 17 00:00:00 2001 From: Savyasachee Jha Date: Wed, 19 May 2021 16:14:16 +0530 Subject: Hardened systemd unit files (#9803) Signed-off-by: Savyasachee Jha savya.jha@hawkradius.com --- changelog.d/9803.doc | 1 + contrib/systemd/override-hardened.conf | 71 ++++++++++++++++++++++++++++++++++ docs/systemd-with-workers/README.md | 30 ++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 changelog.d/9803.doc create mode 100644 contrib/systemd/override-hardened.conf (limited to 'docs') diff --git a/changelog.d/9803.doc b/changelog.d/9803.doc new file mode 100644 index 0000000000..16c7ba7033 --- /dev/null +++ b/changelog.d/9803.doc @@ -0,0 +1 @@ +Add hardened systemd files as proposed in [#9760](https://github.com/matrix-org/synapse/issues/9760) and added them to `contrib/`. Change the docs to reflect the presence of these files. diff --git a/contrib/systemd/override-hardened.conf b/contrib/systemd/override-hardened.conf new file mode 100644 index 0000000000..b2fa3ae7c5 --- /dev/null +++ b/contrib/systemd/override-hardened.conf @@ -0,0 +1,71 @@ +[Service] +# The following directives give the synapse service R/W access to: +# - /run/matrix-synapse +# - /var/lib/matrix-synapse +# - /var/log/matrix-synapse + +RuntimeDirectory=matrix-synapse +StateDirectory=matrix-synapse +LogsDirectory=matrix-synapse + +###################### +## Security Sandbox ## +###################### + +# Make sure that the service has its own unshared tmpfs at /tmp and that it +# cannot see or change any real devices +PrivateTmp=true +PrivateDevices=true + +# We give no capabilities to a service by default +CapabilityBoundingSet= +AmbientCapabilities= + +# Protect the following from modification: +# - The entire filesystem +# - sysctl settings and loaded kernel modules +# - No modifications allowed to Control Groups +# - Hostname +# - System Clock +ProtectSystem=strict +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +ProtectClock=true +ProtectHostname=true + +# Prevent access to the following: +# - /home directory +# - Kernel logs +ProtectHome=tmpfs +ProtectKernelLogs=true + +# Make sure that the process can only see PIDs and process details of itself, +# and the second option disables seeing details of things like system load and +# I/O etc +ProtectProc=invisible +ProcSubset=pid + +# While not needed, we set these options explicitly +# - This process has been given access to the host network +# - It can also communicate with any IP Address +PrivateNetwork=false +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +IPAddressAllow=any + +# Restrict system calls to a sane bunch +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallFilter=~@privileged @resources @obsolete + +# Misc restrictions +# - Since the process is a python process it needs to be able to write and +# execute memory regions, so we set MemoryDenyWriteExecute to false +RestrictSUIDSGID=true +RemoveIPC=true +NoNewPrivileges=true +RestrictRealtime=true +RestrictNamespaces=true +LockPersonality=true +PrivateUsers=true +MemoryDenyWriteExecute=false diff --git a/docs/systemd-with-workers/README.md b/docs/systemd-with-workers/README.md index cfa36be7b4..a1135e9ed5 100644 --- a/docs/systemd-with-workers/README.md +++ b/docs/systemd-with-workers/README.md @@ -65,3 +65,33 @@ systemctl restart matrix-synapse-worker@federation_reader.service systemctl enable matrix-synapse-worker@federation_writer.service systemctl restart matrix-synapse.target ``` + +## Hardening + +**Optional:** If further hardening is desired, the file +`override-hardened.conf` may be copied from +`contrib/systemd/override-hardened.conf` in this repository to the location +`/etc/systemd/system/matrix-synapse.service.d/override-hardened.conf` (the +directory may have to be created). It enables certain sandboxing features in +systemd to further secure the synapse service. You may read the comments to +understand what the override file is doing. The same file will need to be copied +to +`/etc/systemd/system/matrix-synapse-worker@.service.d/override-hardened-worker.conf` +(this directory may also have to be created) in order to apply the same +hardening options to any worker processes. + +Once these files have been copied to their appropriate locations, simply reload +systemd's manager config files and restart all Synapse services to apply the hardening options. They will automatically +be applied at every restart as long as the override files are present at the +specified locations. + +```sh +systemctl daemon-reload + +# Restart services +systemctl restart matrix-synapse.target +``` + +In order to see their effect, you may run `systemd-analyze security +matrix-synapse.service` before and after applying the hardening options to see +the changes being applied at a glance. -- cgit 1.5.1 From 141b073c7bf649ac12b7e12b118770677a64f51f Mon Sep 17 00:00:00 2001 From: Javier Junquera Sánchez Date: Thu, 20 May 2021 15:24:19 +0200 Subject: Update user_directory.md (#10016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Javier Junquera Sánchez --- changelog.d/10016.doc | 1 + docs/user_directory.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10016.doc (limited to 'docs') diff --git a/changelog.d/10016.doc b/changelog.d/10016.doc new file mode 100644 index 0000000000..f9b615d7d7 --- /dev/null +++ b/changelog.d/10016.doc @@ -0,0 +1 @@ +Fix broken link in user directory documentation. Contributed by @junquera. diff --git a/docs/user_directory.md b/docs/user_directory.md index 872fc21979..d4f38d2cf1 100644 --- a/docs/user_directory.md +++ b/docs/user_directory.md @@ -7,6 +7,6 @@ who are present in a publicly viewable room present on the server. The directory info is stored in various tables, which can (typically after DB corruption) get stale or out of sync. If this happens, for now the -solution to fix it is to execute the SQL [here](../synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql) +solution to fix it is to execute the SQL [here](https://github.com/matrix-org/synapse/blob/master/synapse/storage/schema/main/delta/53/user_dir_populate.sql) and then restart synapse. This should then start a background task to flush the current tables and regenerate the directory. -- cgit 1.5.1 From 387c297489b90bd80212ac1391666eecd01ff701 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 24 May 2021 13:37:30 +0200 Subject: Add missing entry to the table of contents of room admin API (#10043) --- changelog.d/10043.doc | 1 + docs/admin_api/rooms.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/10043.doc (limited to 'docs') diff --git a/changelog.d/10043.doc b/changelog.d/10043.doc new file mode 100644 index 0000000000..a574ec0bf0 --- /dev/null +++ b/changelog.d/10043.doc @@ -0,0 +1 @@ +Add missing room state entry to the table of contents of room admin API. \ No newline at end of file diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md index 01d3882426..5721210fee 100644 --- a/docs/admin_api/rooms.md +++ b/docs/admin_api/rooms.md @@ -4,6 +4,7 @@ * [Usage](#usage) - [Room Details API](#room-details-api) - [Room Members API](#room-members-api) +- [Room State API](#room-state-api) - [Delete Room API](#delete-room-api) * [Parameters](#parameters-1) * [Response](#response) -- cgit 1.5.1 From 316f89e87f462608a5e63ba567ad549330f1456a Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 24 May 2021 08:57:14 -0400 Subject: Enable experimental spaces by default. (#10011) The previous spaces_enabled flag now defaults to true and is exposed in the sample config. --- changelog.d/10011.feature | 1 + docs/sample_config.yaml | 15 +++++++++++++++ synapse/config/experimental.py | 19 ++++++++++++++++++- synapse/config/homeserver.py | 2 +- 4 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 changelog.d/10011.feature (limited to 'docs') diff --git a/changelog.d/10011.feature b/changelog.d/10011.feature new file mode 100644 index 0000000000..409140fb13 --- /dev/null +++ b/changelog.d/10011.feature @@ -0,0 +1 @@ +Enable experimental support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) (spaces summary API) and [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) (restricted join rules) by default. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 2952f2ba32..f0f9f06a6e 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -2943,3 +2943,18 @@ redis: # Optional password if configured on the Redis instance # #password: + + +# Enable experimental features in Synapse. +# +# Experimental features might break or be removed without a deprecation +# period. +# +experimental_features: + # Support for Spaces (MSC1772), it enables the following: + # + # * The Spaces Summary API (MSC2946). + # * Restricting room membership based on space membership (MSC3083). + # + # Uncomment to disable support for Spaces. + #spaces_enabled: false diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index a693fba877..cc67377f0f 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -29,9 +29,26 @@ class ExperimentalConfig(Config): self.msc2858_enabled = experimental.get("msc2858_enabled", False) # type: bool # Spaces (MSC1772, MSC2946, MSC3083, etc) - self.spaces_enabled = experimental.get("spaces_enabled", False) # type: bool + self.spaces_enabled = experimental.get("spaces_enabled", True) # type: bool if self.spaces_enabled: KNOWN_ROOM_VERSIONS[RoomVersions.MSC3083.identifier] = RoomVersions.MSC3083 # MSC3026 (busy presence state) self.msc3026_enabled = experimental.get("msc3026_enabled", False) # type: bool + + def generate_config_section(self, **kwargs): + return """\ + # Enable experimental features in Synapse. + # + # Experimental features might break or be removed without a deprecation + # period. + # + experimental_features: + # Support for Spaces (MSC1772), it enables the following: + # + # * The Spaces Summary API (MSC2946). + # * Restricting room membership based on space membership (MSC3083). + # + # Uncomment to disable support for Spaces. + #spaces_enabled: false + """ diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index c23b66c88c..5ae0f55bcc 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -57,7 +57,6 @@ class HomeServerConfig(RootConfig): config_classes = [ ServerConfig, - ExperimentalConfig, TlsConfig, FederationConfig, CacheConfig, @@ -94,4 +93,5 @@ class HomeServerConfig(RootConfig): TracerConfig, WorkerConfig, RedisConfig, + ExperimentalConfig, ] -- cgit 1.5.1 From 057ce7b75406dc97be8ff2c890c47fd9357b0773 Mon Sep 17 00:00:00 2001 From: Jerin J Titus <72017981+jerinjtitus@users.noreply.github.com> Date: Mon, 24 May 2021 22:13:30 +0530 Subject: Remove tls_fingerprints option (#9280) Signed-off-by: Jerin J Titus <72017981+jerinjtitus@users.noreply.github.com> --- changelog.d/9280.removal | 1 + docs/sample_config.yaml | 27 ---------------- scripts-dev/convert_server_keys.py | 7 ----- synapse/config/tls.py | 50 ------------------------------ synapse/rest/key/v2/local_key_resource.py | 8 ----- synapse/rest/key/v2/remote_key_resource.py | 3 -- 6 files changed, 1 insertion(+), 95 deletions(-) create mode 100644 changelog.d/9280.removal (limited to 'docs') diff --git a/changelog.d/9280.removal b/changelog.d/9280.removal new file mode 100644 index 0000000000..c2ed3d308d --- /dev/null +++ b/changelog.d/9280.removal @@ -0,0 +1 @@ +Removed support for the deprecated `tls_fingerprints` configuration setting. Contributed by Jerin J Titus. \ No newline at end of file diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index f0f9f06a6e..6576b153d0 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -683,33 +683,6 @@ acme: # account_key_file: DATADIR/acme_account.key -# List of allowed TLS fingerprints for this server to publish along -# with the signing keys for this server. Other matrix servers that -# make HTTPS requests to this server will check that the TLS -# certificates returned by this server match one of the fingerprints. -# -# Synapse automatically adds the fingerprint of its own certificate -# to the list. So if federation traffic is handled directly by synapse -# then no modification to the list is required. -# -# If synapse is run behind a load balancer that handles the TLS then it -# will be necessary to add the fingerprints of the certificates used by -# the loadbalancers to this list if they are different to the one -# synapse is using. -# -# Homeservers are permitted to cache the list of TLS fingerprints -# returned in the key responses up to the "valid_until_ts" returned in -# key. It may be necessary to publish the fingerprints of a new -# certificate and wait until the "valid_until_ts" of the previous key -# responses have passed before deploying it. -# -# You can calculate a fingerprint from a given TLS listener via: -# openssl s_client -connect $host:$port < /dev/null 2> /dev/null | -# openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '=' -# or by checking matrix.org/federationtester/api/report?server_name=$host -# -#tls_fingerprints: [{"sha256": ""}] - ## Federation ## diff --git a/scripts-dev/convert_server_keys.py b/scripts-dev/convert_server_keys.py index 961dc59f11..d4314a054c 100644 --- a/scripts-dev/convert_server_keys.py +++ b/scripts-dev/convert_server_keys.py @@ -1,4 +1,3 @@ -import hashlib import json import sys import time @@ -54,15 +53,9 @@ def convert_v1_to_v2(server_name, valid_until, keys, certificate): "server_name": server_name, "verify_keys": {key_id: {"key": key} for key_id, key in keys.items()}, "valid_until_ts": valid_until, - "tls_fingerprints": [fingerprint(certificate)], } -def fingerprint(certificate): - finger = hashlib.sha256(certificate) - return {"sha256": encode_base64(finger.digest())} - - def rows_v2(server, json): valid_until = json["valid_until_ts"] key_json = encode_canonical_json(json) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 7df4e4c3e6..26f1150ca5 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -16,11 +16,8 @@ import logging import os import warnings from datetime import datetime -from hashlib import sha256 from typing import List, Optional, Pattern -from unpaddedbase64 import encode_base64 - from OpenSSL import SSL, crypto from twisted.internet._sslverify import Certificate, trustRootFromCertificates @@ -83,13 +80,6 @@ class TlsConfig(Config): "configured." ) - self._original_tls_fingerprints = config.get("tls_fingerprints", []) - - if self._original_tls_fingerprints is None: - self._original_tls_fingerprints = [] - - self.tls_fingerprints = list(self._original_tls_fingerprints) - # Whether to verify certificates on outbound federation traffic self.federation_verify_certificates = config.get( "federation_verify_certificates", True @@ -248,19 +238,6 @@ class TlsConfig(Config): e, ) - self.tls_fingerprints = list(self._original_tls_fingerprints) - - if self.tls_certificate: - # Check that our own certificate is included in the list of fingerprints - # and include it if it is not. - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, self.tls_certificate - ) - sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) - sha256_fingerprints = {f["sha256"] for f in self.tls_fingerprints} - if sha256_fingerprint not in sha256_fingerprints: - self.tls_fingerprints.append({"sha256": sha256_fingerprint}) - def generate_config_section( self, config_dir_path, @@ -443,33 +420,6 @@ class TlsConfig(Config): # If unspecified, we will use CONFDIR/client.key. # account_key_file: %(default_acme_account_file)s - - # List of allowed TLS fingerprints for this server to publish along - # with the signing keys for this server. Other matrix servers that - # make HTTPS requests to this server will check that the TLS - # certificates returned by this server match one of the fingerprints. - # - # Synapse automatically adds the fingerprint of its own certificate - # to the list. So if federation traffic is handled directly by synapse - # then no modification to the list is required. - # - # If synapse is run behind a load balancer that handles the TLS then it - # will be necessary to add the fingerprints of the certificates used by - # the loadbalancers to this list if they are different to the one - # synapse is using. - # - # Homeservers are permitted to cache the list of TLS fingerprints - # returned in the key responses up to the "valid_until_ts" returned in - # key. It may be necessary to publish the fingerprints of a new - # certificate and wait until the "valid_until_ts" of the previous key - # responses have passed before deploying it. - # - # You can calculate a fingerprint from a given TLS listener via: - # openssl s_client -connect $host:$port < /dev/null 2> /dev/null | - # openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '=' - # or by checking matrix.org/federationtester/api/report?server_name=$host - # - #tls_fingerprints: [{"sha256": ""}] """ # Lowercase the string representation of boolean values % { diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index e8dbe240d8..a5fcd15e3a 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -48,11 +48,6 @@ class LocalKey(Resource): "key": # base64 encoded NACL verification key. } }, - "tls_fingerprints": [ # Fingerprints of the TLS certs this server uses. - { - "sha256": # base64 encoded sha256 fingerprint of the X509 cert - }, - ], "signatures": { "this.server.example.com": { "algorithm:version": # NACL signature for this server @@ -89,14 +84,11 @@ class LocalKey(Resource): "expired_ts": key.expired_ts, } - tls_fingerprints = self.config.tls_fingerprints - json_object = { "valid_until_ts": self.valid_until_ts, "server_name": self.config.server_name, "verify_keys": verify_keys, "old_verify_keys": old_verify_keys, - "tls_fingerprints": tls_fingerprints, } for key in self.config.signing_key: json_object = sign_json(json_object, self.config.server_name, key) diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index f648678b09..aba1734a55 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -73,9 +73,6 @@ class RemoteKey(DirectServeJsonResource): "expired_ts": 0, # when the key stop being used. } } - "tls_fingerprints": [ - { "sha256": # fingerprint } - ] "signatures": { "remote.server.example.com": {...} "this.server.example.com": {...} -- cgit 1.5.1