From 4efe1d4d3f61a0ce3769b119d7169c03ac1f9564 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 3 Jan 2020 12:57:24 +0100 Subject: Fix a typo in the purge jobs configuration example --- synapse/config/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 38f6ff9edc..3463d53d10 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -955,7 +955,7 @@ class ServerConfig(Config): #purge_jobs: # - shortest_max_lifetime: 1d # longest_max_lifetime: 3d - # interval: 5m: + # interval: 5m # - shortest_max_lifetime: 3d # longest_max_lifetime: 1y # interval: 24h -- cgit 1.5.1 From dd2954f78dfbea5a24da079e4d9a114f91e55698 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 3 Jan 2020 12:58:12 +0100 Subject: Update sample config --- docs/sample_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index e3b05423b8..cc261d96d0 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -394,7 +394,7 @@ retention: #purge_jobs: # - shortest_max_lifetime: 1d # longest_max_lifetime: 3d - # interval: 5m: + # interval: 5m # - shortest_max_lifetime: 3d # longest_max_lifetime: 1y # interval: 24h -- cgit 1.5.1 From 9c59bc59c8daee1c2e98008e9929ac67cf08ff0b Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 3 Jan 2020 13:00:32 +0100 Subject: Changelog --- changelog.d/6621.doc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/6621.doc diff --git a/changelog.d/6621.doc b/changelog.d/6621.doc new file mode 100644 index 0000000000..6722ccfda3 --- /dev/null +++ b/changelog.d/6621.doc @@ -0,0 +1 @@ +Fix a typo in the configuration example for purge jobs in the sample configuration file. -- cgit 1.5.1 From 9279a2c4e4c1c63e87d43fc9f6ad2c495bf47e67 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 3 Jan 2020 13:43:55 +0100 Subject: Add a complete documentation of the message retention policies support --- changelog.d/6623.doc | 1 + docs/message_retention_policies.md | 191 +++++++++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 changelog.d/6623.doc create mode 100644 docs/message_retention_policies.md diff --git a/changelog.d/6623.doc b/changelog.d/6623.doc new file mode 100644 index 0000000000..c8aade0974 --- /dev/null +++ b/changelog.d/6623.doc @@ -0,0 +1 @@ +Add a complete documentation of the message retention policies support. diff --git a/docs/message_retention_policies.md b/docs/message_retention_policies.md new file mode 100644 index 0000000000..78055b2f64 --- /dev/null +++ b/docs/message_retention_policies.md @@ -0,0 +1,191 @@ +# Message retention policies + +Synapse admins can enable support for message retention policies on +their homeserver. Message retention policies exist at a room level, +follow the semantics described in +[MSC1763](https://github.com/matrix-org/matrix-doc/blob/matthew/msc1763/proposals/1763-configurable-retention-periods.md), +and allow server and room admins to configure how long messages should +be kept in a homeserver's database before being purged from it. + +A message retention policy is mainly defined by its `max_lifetime` +parameter, which defines how long a message can be kept around after +it's been sent in the room. If a room doesn't have a message retention +policy, and there's no default one for a given server, then no message +sent in that room is ever purged on that server. + +MSC1763 also specifies semantics for a `min_lifetime` parameter which +defines the amount of time after which an event _can_ get purged (after +it's been sent to the room), but Synapse doesn't currently support it +beyond registering it. + +Both `max_lifetime` and `min_lifetime` are optional parameters. + +Note that message retention policies don't apply to state events. + +Once an event reaches its expiry date (defined as the time it was sent +plus the value for `max_lifetime` in the room), two things happen: + +* Synapse stops serving the event to clients via any endpoint. +* The message gets picked up by the next purge job (see the "Purge jobs" + section) and is removed from Synapse's database. + +Since purge jobs don't run continuously, this means that an event might +stay in a server's database for longer than the value for `max_lifetime` +in the room would allow, though hidden from clients. + +Similarly, if a server (with support for message retention policies +enabled) receives from another server an event that should have been +purged according to its room's policy, then the receiving server will +process and store that event until it's picked up by the next purge job, +though it will always hide it from clients. + + +## Room configuration + +To configure a room's message retention policy, a room's admin or +moderator needs to send a state event in that room with the type +`m.room.retention` and the following content: + +```json +{ + "max_lifetime": ... +} +``` + +In this event's content, the `max_lifetime` parameter has the same +meaning as previously described, and needs to be expressed in +milliseconds. The event's content can also include a `min_lifetime` +parameter, which has the same meaning and limited support as previously +described. + +Note that over every server in the room, only the ones with support for +message retention policies will actually remove expired events. While +we plan to eventually enable this support by default in Synapse, this +isn't currently the case. + + +## Server configuration + +Support for this feature can be enabled and configured in the +`retention` section of the Synapse configuration file (see the +[sample file](https://github.com/matrix-org/synapse/blob/v1.7.3/docs/sample_config.yaml#L332-L393)). + +To enable support for message retentions policies, set the setting +`enabled` in this section to `true`. + + +### Default policy + +A default message retention policy is a policy defined in Synapse's +configuration that is used by Synapse for every room that doesn't have a +message retention policy configured in its state. This allows server +admins to ensure that messages are never kept indefinitely in a server's +database. + +A default policy can be defined as such, in the `retention` section of +the configuration file: + +```yaml + default_policy: + min_lifetime: 1d + max_lifetime: 1y +``` + +Here, `min_lifetime` and `max_lifetime` have the same meaning and level +of support as previously described. They can be expressed either as a +duration (using the units `s` (seconds), `m` (minutes), `h` (hours), +`d` (days), `w` (weeks) and `y` (years)) or as a number of milliseconds. + + +### Purge jobs + +Purge jobs are the jobs that Synapse run in the background to purge +expired events from the database. They are only run if support for +message retention policies is enabled in the server's configuration. If +no configuration for purge jobs is configured by the server admin, +Synapse will run one daily that will handle every room with a message +retention policy (or, if the server has a default policy configured, +every room it knows), which should be enough in most cases. + +Some server admins might want a finer control on when events are removed +depending on an event's room's policy. This can be done by setting the +`purge_jobs` sub-section in the `retention` section of the configuration +file. An example of such configuration could be: + +```yaml + purge_jobs: + - longest_max_lifetime: 3d + interval: 12h + - shortest_max_lifetime: 3d + longest_max_lifetime: 1w + interval: 1d + - shortest_max_lifetime: 1w + interval: 2d +``` + +In this example, we define two jobs: + +* one that runs twice a day (every 12 hours) and purges events in rooms + which policy's `max_lifetime` is lower or equal to 3 days. +* one that runs once a day and purges events in rooms which policy's + `max_lifetime` is between 3 days and a week. +* one that runs once every 2 days and purges events in rooms which + policy's `max_lifetime` is greater than a week. + +Note that this example is tailored to show different configurations and +features slightly more jobs than it's probably necessary (in practice, a +server admin would probably consider it better to replace the two last +jobs with one that runs once a day and handles rooms which which +policy's `max_lifetime` is greater than 3 days). + +Keep in mind, when configuring these jobs, that a purge job can become +quite heavy on the server if it targets many rooms, therefore prefer +having jobs with a low interval that target a limited set of rooms. Also +make sure to include a job with no minimum and one with no maximum to +make sure your configuration handles every policy. + +As previously mentioned in this documentation, while a purge job that +runs e.g. every day means that an expired event might stay in the +database for up to a day after its expiry, Synapse hides expired events +from clients as soon as they expire, so the event is not visible to +local users between its expiry date and the moment it gets purged from +the server's database. + + +### Lifetime limits + +**Note: this feature is mainly useful within a closed federation or on +servers that don't federate, because there currently is no way to +enforce these limits in an open federation.** + +Server admins can restrict the values their local users are allowed to +use for both `min_lifetime` and `max_lifetime`. These limits can be +defined as such in the `retention` section of the configuration file: + +```yaml + allowed_lifetime_min: 1d + allowed_lifetime_max: 1y +``` + +Here, `allowed_lifetime_min` is the lowest value a local user can set +for both `min_lifetime` and `max_lifetime`, and `allowed_lifetime_max` +is the highest value. Both parameters are optional (e.g. setting +`allowed_lifetime_min` but not `allowed_lifetime_max` only enforces a +minimum and no maximum). + +Like other settings in this section, these parameters can be expressed +either as a duration or as a number of milliseconds. + + +## Note on reclaiming disk space + +While purge jobs actually delete data from the database, the disk space +used by the database might not decrease immediately on the database's +host. However, even though the database engine won't free up the disk +space, it will start writing new data into where the purged data was. + +If you want to reclaim the freed disk space anyway and return it to the +operating system, the server admin needs to run `VACUUM FULL;` on the +database (see the related +[PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-vacuum.html)). + -- cgit 1.5.1 From 51b8a21f0c3f52c26c63c196f5ed11b8be2394af Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 3 Jan 2020 13:49:12 +0100 Subject: Rename changelog --- changelog.d/6623.doc | 1 - changelog.d/6624.doc | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog.d/6623.doc create mode 100644 changelog.d/6624.doc diff --git a/changelog.d/6623.doc b/changelog.d/6623.doc deleted file mode 100644 index c8aade0974..0000000000 --- a/changelog.d/6623.doc +++ /dev/null @@ -1 +0,0 @@ -Add a complete documentation of the message retention policies support. diff --git a/changelog.d/6624.doc b/changelog.d/6624.doc new file mode 100644 index 0000000000..c8aade0974 --- /dev/null +++ b/changelog.d/6624.doc @@ -0,0 +1 @@ +Add a complete documentation of the message retention policies support. -- cgit 1.5.1 From b7dec300b7419402a0d5fc00e34684b95618a7d9 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 3 Jan 2020 13:51:59 +0100 Subject: Fix vacuum instructions for sqlite --- docs/message_retention_policies.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/message_retention_policies.md b/docs/message_retention_policies.md index 78055b2f64..72f08fbb4c 100644 --- a/docs/message_retention_policies.md +++ b/docs/message_retention_policies.md @@ -185,7 +185,7 @@ host. However, even though the database engine won't free up the disk space, it will start writing new data into where the purged data was. If you want to reclaim the freed disk space anyway and return it to the -operating system, the server admin needs to run `VACUUM FULL;` on the -database (see the related +operating system, the server admin needs to run `VACUUM FULL;` (or +`VACUUM;` for SQLite databases) on Synapse's database (see the related [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-vacuum.html)). -- cgit 1.5.1 From 3a864771624313dde75dfd1fa50dfca5cbadc8ca Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 7 Jan 2020 14:53:07 +0000 Subject: Change the example from 5min to 12h Have a purge job running every 5min is probably not something we want to advise admins to do as a sort-of default. --- docs/sample_config.yaml | 8 ++++---- synapse/config/server.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index cc261d96d0..77a3fe6cd9 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -387,17 +387,17 @@ retention: # # The rationale for this per-job configuration is that some rooms might have a # retention policy with a low 'max_lifetime', where history needs to be purged - # of outdated messages on a very frequent basis (e.g. every 5min), but not want + # of outdated messages on a more frequent basis (e.g. every 12h), but not want # that purge to be performed by a job that's iterating over every room it knows, - # which would be quite heavy on the server. + # which could be heavy on the server. # #purge_jobs: # - shortest_max_lifetime: 1d # longest_max_lifetime: 3d - # interval: 5m + # interval: 12h # - shortest_max_lifetime: 3d # longest_max_lifetime: 1y - # interval: 24h + # interval: 1d ## TLS ## diff --git a/synapse/config/server.py b/synapse/config/server.py index 3463d53d10..11ff559224 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -948,17 +948,17 @@ class ServerConfig(Config): # # The rationale for this per-job configuration is that some rooms might have a # retention policy with a low 'max_lifetime', where history needs to be purged - # of outdated messages on a very frequent basis (e.g. every 5min), but not want + # of outdated messages on a more frequent basis (e.g. every 12h), but not want # that purge to be performed by a job that's iterating over every room it knows, - # which would be quite heavy on the server. + # which could be heavy on the server. # #purge_jobs: # - shortest_max_lifetime: 1d # longest_max_lifetime: 3d - # interval: 5m + # interval: 12h # - shortest_max_lifetime: 3d # longest_max_lifetime: 1y - # interval: 24h + # interval: 1d """ % locals() ) -- cgit 1.5.1 From 391fb4779106a291724137e6c52494308729ffcb Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 7 Jan 2020 14:54:32 +0000 Subject: Reword --- docs/sample_config.yaml | 6 +++--- synapse/config/server.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 77a3fe6cd9..cec6b3e544 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -387,9 +387,9 @@ retention: # # The rationale for this per-job configuration is that some rooms might have a # retention policy with a low 'max_lifetime', where history needs to be purged - # of outdated messages on a more frequent basis (e.g. every 12h), but not want - # that purge to be performed by a job that's iterating over every room it knows, - # which could be heavy on the server. + # of outdated messages on a more frequent basis than for the rest of the rooms + # (e.g. every 12h), but not want that purge to be performed by a job that's + # iterating over every room it knows, which could be heavy on the server. # #purge_jobs: # - shortest_max_lifetime: 1d diff --git a/synapse/config/server.py b/synapse/config/server.py index 11ff559224..9ac112233b 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -948,9 +948,9 @@ class ServerConfig(Config): # # The rationale for this per-job configuration is that some rooms might have a # retention policy with a low 'max_lifetime', where history needs to be purged - # of outdated messages on a more frequent basis (e.g. every 12h), but not want - # that purge to be performed by a job that's iterating over every room it knows, - # which could be heavy on the server. + # of outdated messages on a more frequent basis than for the rest of the rooms + # (e.g. every 12h), but not want that purge to be performed by a job that's + # iterating over every room it knows, which could be heavy on the server. # #purge_jobs: # - shortest_max_lifetime: 1d -- cgit 1.5.1 From 03edfc58500197fee40c808680551ea55d1560e8 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 7 Jan 2020 15:59:05 +0100 Subject: Update changelog.d/6624.doc Co-Authored-By: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- changelog.d/6624.doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/6624.doc b/changelog.d/6624.doc index c8aade0974..bc9a022db2 100644 --- a/changelog.d/6624.doc +++ b/changelog.d/6624.doc @@ -1 +1 @@ -Add a complete documentation of the message retention policies support. +Add complete documentation of the message retention policies support. -- cgit 1.5.1 From 01fbd9573626381c51700845956c9c9451cb645a Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 7 Jan 2020 15:59:38 +0100 Subject: Apply suggestions from code review Co-Authored-By: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- docs/message_retention_policies.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/message_retention_policies.md b/docs/message_retention_policies.md index 72f08fbb4c..84f0925230 100644 --- a/docs/message_retention_policies.md +++ b/docs/message_retention_policies.md @@ -9,13 +9,13 @@ be kept in a homeserver's database before being purged from it. A message retention policy is mainly defined by its `max_lifetime` parameter, which defines how long a message can be kept around after -it's been sent in the room. If a room doesn't have a message retention +it was sent to the room. If a room doesn't have a message retention policy, and there's no default one for a given server, then no message sent in that room is ever purged on that server. MSC1763 also specifies semantics for a `min_lifetime` parameter which defines the amount of time after which an event _can_ get purged (after -it's been sent to the room), but Synapse doesn't currently support it +it was sent to the room), but Synapse doesn't currently support it beyond registering it. Both `max_lifetime` and `min_lifetime` are optional parameters. @@ -70,7 +70,7 @@ Support for this feature can be enabled and configured in the `retention` section of the Synapse configuration file (see the [sample file](https://github.com/matrix-org/synapse/blob/v1.7.3/docs/sample_config.yaml#L332-L393)). -To enable support for message retentions policies, set the setting +To enable support for message retention policies, set the setting `enabled` in this section to `true`. @@ -99,7 +99,7 @@ duration (using the units `s` (seconds), `m` (minutes), `h` (hours), ### Purge jobs -Purge jobs are the jobs that Synapse run in the background to purge +Purge jobs are the jobs that Synapse runs in the background to purge expired events from the database. They are only run if support for message retention policies is enabled in the server's configuration. If no configuration for purge jobs is configured by the server admin, @@ -188,4 +188,3 @@ If you want to reclaim the freed disk space anyway and return it to the operating system, the server admin needs to run `VACUUM FULL;` (or `VACUUM;` for SQLite databases) on Synapse's database (see the related [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-vacuum.html)). - -- cgit 1.5.1 From 7ba98a2874c6c14c3f2ceb9b633a13d3e7345065 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 7 Jan 2020 15:14:33 +0000 Subject: Incorporate review --- docs/message_retention_policies.md | 55 +++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/message_retention_policies.md b/docs/message_retention_policies.md index 84f0925230..42b637516f 100644 --- a/docs/message_retention_policies.md +++ b/docs/message_retention_policies.md @@ -6,6 +6,9 @@ follow the semantics described in [MSC1763](https://github.com/matrix-org/matrix-doc/blob/matthew/msc1763/proposals/1763-configurable-retention-periods.md), and allow server and room admins to configure how long messages should be kept in a homeserver's database before being purged from it. +**Please note that, as this feature isn't part of the Matrix +specification yet, this implementation is to be considered as +experimental.** A message retention policy is mainly defined by its `max_lifetime` parameter, which defines how long a message can be kept around after @@ -40,30 +43,6 @@ process and store that event until it's picked up by the next purge job, though it will always hide it from clients. -## Room configuration - -To configure a room's message retention policy, a room's admin or -moderator needs to send a state event in that room with the type -`m.room.retention` and the following content: - -```json -{ - "max_lifetime": ... -} -``` - -In this event's content, the `max_lifetime` parameter has the same -meaning as previously described, and needs to be expressed in -milliseconds. The event's content can also include a `min_lifetime` -parameter, which has the same meaning and limited support as previously -described. - -Note that over every server in the room, only the ones with support for -message retention policies will actually remove expired events. While -we plan to eventually enable this support by default in Synapse, this -isn't currently the case. - - ## Server configuration Support for this feature can be enabled and configured in the @@ -103,9 +82,8 @@ Purge jobs are the jobs that Synapse runs in the background to purge expired events from the database. They are only run if support for message retention policies is enabled in the server's configuration. If no configuration for purge jobs is configured by the server admin, -Synapse will run one daily that will handle every room with a message -retention policy (or, if the server has a default policy configured, -every room it knows), which should be enough in most cases. +Synapse will use a default configuration, which is described in the +[sample configuration file](https://github.com/matrix-org/synapse/blob/v1.7.3/docs/sample_config.yaml#L332-L393). Some server admins might want a finer control on when events are removed depending on an event's room's policy. This can be done by setting the @@ -177,6 +155,29 @@ Like other settings in this section, these parameters can be expressed either as a duration or as a number of milliseconds. +## Room configuration + +To configure a room's message retention policy, a room's admin or +moderator needs to send a state event in that room with the type +`m.room.retention` and the following content: + +```json +{ + "max_lifetime": ... +} +``` + +In this event's content, the `max_lifetime` parameter has the same +meaning as previously described, and needs to be expressed in +milliseconds. The event's content can also include a `min_lifetime` +parameter, which has the same meaning and limited support as previously +described. + +Note that over every server in the room, only the ones with support for +message retention policies will actually remove expired events. This +support is currently not enabled by default in Synapse. + + ## Note on reclaiming disk space While purge jobs actually delete data from the database, the disk space -- cgit 1.5.1 From 3675fb9bc6b0f9fd068bee443c1499042359ee99 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 7 Jan 2020 15:15:16 +0000 Subject: Fix reference --- docs/message_retention_policies.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/message_retention_policies.md b/docs/message_retention_policies.md index 42b637516f..c4888c81be 100644 --- a/docs/message_retention_policies.md +++ b/docs/message_retention_policies.md @@ -83,7 +83,7 @@ expired events from the database. They are only run if support for message retention policies is enabled in the server's configuration. If no configuration for purge jobs is configured by the server admin, Synapse will use a default configuration, which is described in the -[sample configuration file](https://github.com/matrix-org/synapse/blob/v1.7.3/docs/sample_config.yaml#L332-L393). +[sample configuration file](https://github.com/matrix-org/synapse/blob/master/docs/sample_config.yaml#L332-L393). Some server admins might want a finer control on when events are removed depending on an event's room's policy. This can be done by setting the -- cgit 1.5.1 From be29ed7ad86a150f603722d7dc307b71f7e98726 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Jan 2020 15:36:41 +0000 Subject: Correctly proxy remote group HTTP errors. (#6654) e.g. if remote returns a 404 then that shouldn't be treated as an error but should be proxied through. --- changelog.d/6654.bugfix | 1 + synapse/handlers/groups_local.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 changelog.d/6654.bugfix diff --git a/changelog.d/6654.bugfix b/changelog.d/6654.bugfix new file mode 100644 index 0000000000..fed35252db --- /dev/null +++ b/changelog.d/6654.bugfix @@ -0,0 +1 @@ +Correctly proxy HTTP errors due to API calls to remote group servers. diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index 92fecbfc44..319565510f 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -130,6 +130,8 @@ class GroupsLocalHandler(object): res = yield self.transport_client.get_group_summary( get_domain_from_id(group_id), group_id, requester_user_id ) + except HttpResponseException as e: + raise e.to_synapse_error() except RequestSendFailed: raise SynapseError(502, "Failed to contact group server") @@ -190,6 +192,8 @@ class GroupsLocalHandler(object): res = yield self.transport_client.create_group( get_domain_from_id(group_id), group_id, user_id, content ) + except HttpResponseException as e: + raise e.to_synapse_error() except RequestSendFailed: raise SynapseError(502, "Failed to contact group server") @@ -231,6 +235,8 @@ class GroupsLocalHandler(object): res = yield self.transport_client.get_users_in_group( get_domain_from_id(group_id), group_id, requester_user_id ) + except HttpResponseException as e: + raise e.to_synapse_error() except RequestSendFailed: raise SynapseError(502, "Failed to contact group server") @@ -271,6 +277,8 @@ class GroupsLocalHandler(object): res = yield self.transport_client.join_group( get_domain_from_id(group_id), group_id, user_id, content ) + except HttpResponseException as e: + raise e.to_synapse_error() except RequestSendFailed: raise SynapseError(502, "Failed to contact group server") @@ -315,6 +323,8 @@ class GroupsLocalHandler(object): res = yield self.transport_client.accept_group_invite( get_domain_from_id(group_id), group_id, user_id, content ) + except HttpResponseException as e: + raise e.to_synapse_error() except RequestSendFailed: raise SynapseError(502, "Failed to contact group server") @@ -361,6 +371,8 @@ class GroupsLocalHandler(object): requester_user_id, content, ) + except HttpResponseException as e: + raise e.to_synapse_error() except RequestSendFailed: raise SynapseError(502, "Failed to contact group server") @@ -424,6 +436,8 @@ class GroupsLocalHandler(object): user_id, content, ) + except HttpResponseException as e: + raise e.to_synapse_error() except RequestSendFailed: raise SynapseError(502, "Failed to contact group server") @@ -460,6 +474,8 @@ class GroupsLocalHandler(object): bulk_result = yield self.transport_client.bulk_get_publicised_groups( get_domain_from_id(user_id), [user_id] ) + except HttpResponseException as e: + raise e.to_synapse_error() except RequestSendFailed: raise SynapseError(502, "Failed to contact group server") -- cgit 1.5.1 From 91718b3f23031e93341b53e0c9b6370f35838a8b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 7 Jan 2020 15:46:04 +0000 Subject: typo --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index df94f742c0..24da66c596 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ Features - Add v2 APIs for the `send_join` and `send_leave` federation endpoints (as described in [MSC1802](https://github.com/matrix-org/matrix-doc/pull/1802)). ([\#6349](https://github.com/matrix-org/synapse/issues/6349)) - Add a develop script to generate full SQL schemas. ([\#6394](https://github.com/matrix-org/synapse/issues/6394)) -- Add custom SAML username mapping functinality through an external provider plugin. ([\#6411](https://github.com/matrix-org/synapse/issues/6411)) +- Add custom SAML username mapping functionality through an external provider plugin. ([\#6411](https://github.com/matrix-org/synapse/issues/6411)) - Automatically delete empty groups/communities. ([\#6453](https://github.com/matrix-org/synapse/issues/6453)) - Add option `limit_profile_requests_to_users_who_share_rooms` to prevent requirement of a local user sharing a room with another user to query their profile information. ([\#6523](https://github.com/matrix-org/synapse/issues/6523)) - Add an `export_signing_key` script to extract the public part of signing keys when rotating them. ([\#6546](https://github.com/matrix-org/synapse/issues/6546)) -- cgit 1.5.1 From dd57715de2b9a0742b38aaab63893e2495b68841 Mon Sep 17 00:00:00 2001 From: Fabian Meyer Date: Wed, 8 Jan 2020 08:25:05 +0100 Subject: contrib/docker-compose: fixing mount that overrides containers' /etc (#6656) The mount in the form of ./matrix-config:/etc overwrites the contents of the container /etc folder. Since all valid ca certificates are stored in /etc, the synapse.push.httppusher, for example, cannot validate the certificate from matrix.org. --- changelog.d/6656.doc | 1 + contrib/docker/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/6656.doc diff --git a/changelog.d/6656.doc b/changelog.d/6656.doc new file mode 100644 index 0000000000..9f32da1a88 --- /dev/null +++ b/changelog.d/6656.doc @@ -0,0 +1 @@ +No more overriding the entire /etc folder of the container in docker-compose.yaml. Contributed by Fabian Meyer. diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 72c87054e5..2b044baf78 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -18,7 +18,7 @@ services: - SYNAPSE_CONFIG_PATH=/etc/homeserver.yaml volumes: # You may either store all the files in a local folder - - ./matrix-config:/etc + - ./matrix-config/homeserver.yaml:/etc/homeserver.yaml - ./files:/data # .. or you may split this between different storage points # - ./files:/data -- cgit 1.5.1 From 3889fcd9d7074c228b90726d97b4ecf372ecc117 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 8 Jan 2020 13:27:29 +0000 Subject: Fix typo in message retention policies doc --- docs/message_retention_policies.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/message_retention_policies.md b/docs/message_retention_policies.md index c4888c81be..4300809dfe 100644 --- a/docs/message_retention_policies.md +++ b/docs/message_retention_policies.md @@ -101,7 +101,7 @@ file. An example of such configuration could be: interval: 2d ``` -In this example, we define two jobs: +In this example, we define three jobs: * one that runs twice a day (every 12 hours) and purges events in rooms which policy's `max_lifetime` is lower or equal to 3 days. -- cgit 1.5.1 From 32ad2a3349e262a431aa9c57fef2d89f629aac31 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 8 Jan 2020 13:28:12 +0000 Subject: Changelog --- changelog.d/6665.doc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/6665.doc diff --git a/changelog.d/6665.doc b/changelog.d/6665.doc new file mode 100644 index 0000000000..bc9a022db2 --- /dev/null +++ b/changelog.d/6665.doc @@ -0,0 +1 @@ +Add complete documentation of the message retention policies support. -- cgit 1.5.1 From 24b2c940fb657270bfbe3d7a18c5e9363c42663d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jan 2020 11:39:29 +0000 Subject: 1.8.0 --- CHANGES.md | 10 ++++++++++ changelog.d/6563.bugfix | 1 - changelog.d/6657.bugfix | 1 - debian/changelog | 8 ++++++-- synapse/__init__.py | 2 +- 5 files changed, 17 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/6563.bugfix delete mode 100644 changelog.d/6657.bugfix diff --git a/CHANGES.md b/CHANGES.md index df94f742c0..e33e0d7f07 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,13 @@ +Synapse 1.8.0 (2020-01-09) +========================== + +Bugfixes +-------- + +- Fix GET request on /_synapse/admin/v2/users endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6563](https://github.com/matrix-org/synapse/issues/6563)) +- Fix incorrect signing of responses from the key server implementation. ([\#6657](https://github.com/matrix-org/synapse/issues/6657)) + + Synapse 1.8.0rc1 (2020-01-07) ============================= diff --git a/changelog.d/6563.bugfix b/changelog.d/6563.bugfix deleted file mode 100644 index 3325fb1dcf..0000000000 --- a/changelog.d/6563.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix GET request on /_synapse/admin/v2/users endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. \ No newline at end of file diff --git a/changelog.d/6657.bugfix b/changelog.d/6657.bugfix deleted file mode 100644 index 94e51a9896..0000000000 --- a/changelog.d/6657.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect signing of responses from the key server implementation. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 75fe89fa97..7413c238e6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,12 @@ -matrix-synapse-py3 (1.7.3ubuntu1) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.8.0) stable; urgency=medium + [ Richard van der Hoff ] * Automate generation of the default log configuration file. - -- Richard van der Hoff Fri, 03 Jan 2020 13:55:38 +0000 + [ Synapse Packaging team ] + * New synapse release 1.8.0. + + -- Synapse Packaging team Thu, 09 Jan 2020 11:39:27 +0000 matrix-synapse-py3 (1.7.3) stable; urgency=medium diff --git a/synapse/__init__.py b/synapse/__init__.py index a3bd855045..0dd538d804 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -36,7 +36,7 @@ try: except ImportError: pass -__version__ = "1.8.0rc1" +__version__ = "1.8.0" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when -- cgit 1.5.1 From 9dfcf47e9bb323f0597ebf8f34a1bcc9f14a02a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jan 2020 11:40:27 +0000 Subject: Fixup changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index e33e0d7f07..690cbdaae9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ Synapse 1.8.0 (2020-01-09) Bugfixes -------- -- Fix GET request on /_synapse/admin/v2/users endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6563](https://github.com/matrix-org/synapse/issues/6563)) +- Fix `GET` request on `/_synapse/admin/v2/users` endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6563](https://github.com/matrix-org/synapse/issues/6563)) - Fix incorrect signing of responses from the key server implementation. ([\#6657](https://github.com/matrix-org/synapse/issues/6657)) -- cgit 1.5.1 From d2906fe6667d3384f37ef03ca87172d643d49587 Mon Sep 17 00:00:00 2001 From: Manuel Stahl <37705355+awesome-manuel@users.noreply.github.com> Date: Thu, 9 Jan 2020 14:31:00 +0100 Subject: Allow admin users to create or modify users without a shared secret (#6495) Signed-off-by: Manuel Stahl --- changelog.d/5742.feature | 1 + docs/admin_api/user_admin_api.rst | 33 +- synapse/handlers/admin.py | 9 + synapse/rest/admin/__init__.py | 2 + synapse/rest/admin/users.py | 142 +++++++ synapse/storage/data_stores/main/registration.py | 2 + tests/rest/admin/test_admin.py | 338 ---------------- tests/rest/admin/test_user.py | 465 +++++++++++++++++++++++ tests/storage/test_registration.py | 2 + 9 files changed, 655 insertions(+), 339 deletions(-) create mode 100644 changelog.d/5742.feature create mode 100644 tests/rest/admin/test_user.py diff --git a/changelog.d/5742.feature b/changelog.d/5742.feature new file mode 100644 index 0000000000..de10302275 --- /dev/null +++ b/changelog.d/5742.feature @@ -0,0 +1 @@ +Allow admin to create or modify a user. Contributed by Awesome Technologies Innovationslabor GmbH. diff --git a/docs/admin_api/user_admin_api.rst b/docs/admin_api/user_admin_api.rst index b451dc5014..0b3d09d694 100644 --- a/docs/admin_api/user_admin_api.rst +++ b/docs/admin_api/user_admin_api.rst @@ -1,3 +1,33 @@ +Create or modify Account +======================== + +This API allows an administrator to create or modify a user account with a +specific ``user_id``. + +This api is:: + + PUT /_synapse/admin/v2/users/ + +with a body of: + +.. code:: json + + { + "password": "user_password", + "displayname": "User", + "avatar_url": "", + "admin": false, + "deactivated": false + } + +including an ``access_token`` of a server admin. + +The parameter ``displayname`` is optional and defaults to ``user_id``. +The parameter ``avatar_url`` is optional. +The parameter ``admin`` is optional and defaults to 'false'. +The parameter ``deactivated`` is optional and defaults to 'false'. +If the user already exists then optional parameters default to the current value. + List Accounts ============= @@ -50,7 +80,8 @@ This API returns information about a specific user account. The api is:: - GET /_synapse/admin/v1/whois/ + GET /_synapse/admin/v1/whois/ (deprecated) + GET /_synapse/admin/v2/users/ including an ``access_token`` of a server admin. diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 1a4ba12385..76d18a8ba8 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -51,6 +51,15 @@ class AdminHandler(BaseHandler): return ret + async def get_user(self, user): + """Function to get user details""" + ret = await self.store.get_user_by_id(user.to_string()) + if ret: + profile = await self.store.get_profileinfo(user.localpart) + ret["displayname"] = profile.display_name + ret["avatar_url"] = profile.avatar_url + return ret + async def get_users(self): """Function to retrieve a list of users in users table. diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index c122c449f4..a10b4a9b72 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -38,6 +38,7 @@ from synapse.rest.admin.users import ( SearchUsersRestServlet, UserAdminServlet, UserRegisterServlet, + UserRestServletV2, UsersRestServlet, UsersRestServletV2, WhoisRestServlet, @@ -191,6 +192,7 @@ def register_servlets(hs, http_server): SendServerNoticeServlet(hs).register(http_server) VersionServlet(hs).register(http_server) UserAdminServlet(hs).register(http_server) + UserRestServletV2(hs).register(http_server) UsersRestServletV2(hs).register(http_server) diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 1937879dbe..574cb90c74 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -102,6 +102,148 @@ class UsersRestServletV2(RestServlet): return 200, ret +class UserRestServletV2(RestServlet): + PATTERNS = (re.compile("^/_synapse/admin/v2/users/(?P@[^/]+)$"),) + + """Get request to list user details. + This needs user to have administrator access in Synapse. + + GET /_synapse/admin/v2/users/ + + returns: + 200 OK with user details if success otherwise an error. + + Put request to allow an administrator to add or modify a user. + This needs user to have administrator access in Synapse. + We use PUT instead of POST since we already know the id of the user + object to create. POST could be used to create guests. + + PUT /_synapse/admin/v2/users/ + { + "password": "secret", + "displayname": "User" + } + + returns: + 201 OK with new user object if user was created or + 200 OK with modified user object if user was modified + otherwise an error. + """ + + def __init__(self, hs): + self.hs = hs + self.auth = hs.get_auth() + self.admin_handler = hs.get_handlers().admin_handler + self.profile_handler = hs.get_profile_handler() + self.set_password_handler = hs.get_set_password_handler() + self.deactivate_account_handler = hs.get_deactivate_account_handler() + self.registration_handler = hs.get_registration_handler() + + async def on_GET(self, request, user_id): + await assert_requester_is_admin(self.auth, request) + + target_user = UserID.from_string(user_id) + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only lookup local users") + + ret = await self.admin_handler.get_user(target_user) + + return 200, ret + + async def on_PUT(self, request, user_id): + await assert_requester_is_admin(self.auth, request) + + target_user = UserID.from_string(user_id) + body = parse_json_object_from_request(request) + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "This endpoint can only be used with local users") + + user = await self.admin_handler.get_user(target_user) + + if user: # modify user + requester = await self.auth.get_user_by_req(request) + + if "displayname" in body: + await self.profile_handler.set_displayname( + target_user, requester, body["displayname"], True + ) + + if "avatar_url" in body: + await self.profile_handler.set_avatar_url( + target_user, requester, body["avatar_url"], True + ) + + if "admin" in body: + set_admin_to = bool(body["admin"]) + if set_admin_to != user["admin"]: + auth_user = requester.user + if target_user == auth_user and not set_admin_to: + raise SynapseError(400, "You may not demote yourself.") + + await self.admin_handler.set_user_server_admin( + target_user, set_admin_to + ) + + if "password" in body: + if ( + not isinstance(body["password"], text_type) + or len(body["password"]) > 512 + ): + raise SynapseError(400, "Invalid password") + else: + new_password = body["password"] + await self._set_password_handler.set_password( + target_user, new_password, requester + ) + + if "deactivated" in body: + deactivate = bool(body["deactivated"]) + if deactivate and not user["deactivated"]: + result = await self.deactivate_account_handler.deactivate_account( + target_user.to_string(), False + ) + if not result: + raise SynapseError(500, "Could not deactivate user") + + user = await self.admin_handler.get_user(target_user) + return 200, user + + else: # create user + if "password" not in body: + raise SynapseError( + 400, "password must be specified", errcode=Codes.BAD_JSON + ) + elif ( + not isinstance(body["password"], text_type) + or len(body["password"]) > 512 + ): + raise SynapseError(400, "Invalid password") + + admin = body.get("admin", None) + user_type = body.get("user_type", None) + displayname = body.get("displayname", None) + + if user_type is not None and user_type not in UserTypes.ALL_USER_TYPES: + raise SynapseError(400, "Invalid user type") + + user_id = await self.registration_handler.register_user( + localpart=target_user.localpart, + password=body["password"], + admin=bool(admin), + default_display_name=displayname, + user_type=user_type, + ) + if "avatar_url" in body: + await self.profile_handler.set_avatar_url( + user_id, requester, body["avatar_url"], True + ) + + ret = await self.admin_handler.get_user(target_user) + + return 201, ret + + class UserRegisterServlet(RestServlet): """ Attributes: diff --git a/synapse/storage/data_stores/main/registration.py b/synapse/storage/data_stores/main/registration.py index 5e8ecac0ea..cb4b2b39a0 100644 --- a/synapse/storage/data_stores/main/registration.py +++ b/synapse/storage/data_stores/main/registration.py @@ -52,11 +52,13 @@ class RegistrationWorkerStore(SQLBaseStore): "name", "password_hash", "is_guest", + "admin", "consent_version", "consent_server_notice_sent", "appservice_id", "creation_ts", "user_type", + "deactivated", ], allow_none=True, desc="get_user_by_id", diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 325bd6a608..6ceb483aa8 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -13,14 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib -import hmac import json from mock import Mock import synapse.rest.admin -from synapse.api.constants import UserTypes from synapse.http.server import JsonResource from synapse.rest.admin import VersionServlet from synapse.rest.client.v1 import events, login, room @@ -47,341 +44,6 @@ class VersionTestCase(unittest.HomeserverTestCase): ) -class UserRegisterTestCase(unittest.HomeserverTestCase): - - servlets = [synapse.rest.admin.register_servlets_for_client_rest_resource] - - def make_homeserver(self, reactor, clock): - - self.url = "/_matrix/client/r0/admin/register" - - self.registration_handler = Mock() - self.identity_handler = Mock() - self.login_handler = Mock() - self.device_handler = Mock() - self.device_handler.check_device_registered = Mock(return_value="FAKE") - - self.datastore = Mock(return_value=Mock()) - self.datastore.get_current_state_deltas = Mock(return_value=(0, [])) - - self.secrets = Mock() - - self.hs = self.setup_test_homeserver() - - self.hs.config.registration_shared_secret = "shared" - - self.hs.get_media_repository = Mock() - self.hs.get_deactivate_account_handler = Mock() - - return self.hs - - def test_disabled(self): - """ - If there is no shared secret, registration through this method will be - prevented. - """ - self.hs.config.registration_shared_secret = None - - request, channel = self.make_request("POST", self.url, b"{}") - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual( - "Shared secret registration is not enabled", channel.json_body["error"] - ) - - def test_get_nonce(self): - """ - Calling GET on the endpoint will return a randomised nonce, using the - homeserver's secrets provider. - """ - secrets = Mock() - secrets.token_hex = Mock(return_value="abcd") - - self.hs.get_secrets = Mock(return_value=secrets) - - request, channel = self.make_request("GET", self.url) - self.render(request) - - self.assertEqual(channel.json_body, {"nonce": "abcd"}) - - def test_expired_nonce(self): - """ - Calling GET on the endpoint will return a randomised nonce, which will - only last for SALT_TIMEOUT (60s). - """ - request, channel = self.make_request("GET", self.url) - self.render(request) - nonce = channel.json_body["nonce"] - - # 59 seconds - self.reactor.advance(59) - - body = json.dumps({"nonce": nonce}) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("username must be specified", channel.json_body["error"]) - - # 61 seconds - self.reactor.advance(2) - - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("unrecognised nonce", channel.json_body["error"]) - - def test_register_incorrect_nonce(self): - """ - Only the provided nonce can be used, as it's checked in the MAC. - """ - request, channel = self.make_request("GET", self.url) - self.render(request) - nonce = channel.json_body["nonce"] - - want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) - want_mac.update(b"notthenonce\x00bob\x00abc123\x00admin") - want_mac = want_mac.hexdigest() - - body = json.dumps( - { - "nonce": nonce, - "username": "bob", - "password": "abc123", - "admin": True, - "mac": want_mac, - } - ) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("HMAC incorrect", channel.json_body["error"]) - - def test_register_correct_nonce(self): - """ - When the correct nonce is provided, and the right key is provided, the - user is registered. - """ - request, channel = self.make_request("GET", self.url) - self.render(request) - nonce = channel.json_body["nonce"] - - want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) - want_mac.update( - nonce.encode("ascii") + b"\x00bob\x00abc123\x00admin\x00support" - ) - want_mac = want_mac.hexdigest() - - body = json.dumps( - { - "nonce": nonce, - "username": "bob", - "password": "abc123", - "admin": True, - "user_type": UserTypes.SUPPORT, - "mac": want_mac, - } - ) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("@bob:test", channel.json_body["user_id"]) - - def test_nonce_reuse(self): - """ - A valid unrecognised nonce. - """ - request, channel = self.make_request("GET", self.url) - self.render(request) - nonce = channel.json_body["nonce"] - - want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) - want_mac.update(nonce.encode("ascii") + b"\x00bob\x00abc123\x00admin") - want_mac = want_mac.hexdigest() - - body = json.dumps( - { - "nonce": nonce, - "username": "bob", - "password": "abc123", - "admin": True, - "mac": want_mac, - } - ) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("@bob:test", channel.json_body["user_id"]) - - # Now, try and reuse it - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("unrecognised nonce", channel.json_body["error"]) - - def test_missing_parts(self): - """ - Synapse will complain if you don't give nonce, username, password, and - mac. Admin and user_types are optional. Additional checks are done for length - and type. - """ - - def nonce(): - request, channel = self.make_request("GET", self.url) - self.render(request) - return channel.json_body["nonce"] - - # - # Nonce check - # - - # Must be present - body = json.dumps({}) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("nonce must be specified", channel.json_body["error"]) - - # - # Username checks - # - - # Must be present - body = json.dumps({"nonce": nonce()}) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("username must be specified", channel.json_body["error"]) - - # Must be a string - body = json.dumps({"nonce": nonce(), "username": 1234}) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("Invalid username", channel.json_body["error"]) - - # Must not have null bytes - body = json.dumps({"nonce": nonce(), "username": "abcd\u0000"}) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("Invalid username", channel.json_body["error"]) - - # Must not have null bytes - body = json.dumps({"nonce": nonce(), "username": "a" * 1000}) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("Invalid username", channel.json_body["error"]) - - # - # Password checks - # - - # Must be present - body = json.dumps({"nonce": nonce(), "username": "a"}) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("password must be specified", channel.json_body["error"]) - - # Must be a string - body = json.dumps({"nonce": nonce(), "username": "a", "password": 1234}) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("Invalid password", channel.json_body["error"]) - - # Must not have null bytes - body = json.dumps({"nonce": nonce(), "username": "a", "password": "abcd\u0000"}) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("Invalid password", channel.json_body["error"]) - - # Super long - body = json.dumps({"nonce": nonce(), "username": "a", "password": "A" * 1000}) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("Invalid password", channel.json_body["error"]) - - # - # user_type check - # - - # Invalid user_type - body = json.dumps( - { - "nonce": nonce(), - "username": "a", - "password": "1234", - "user_type": "invalid", - } - ) - request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) - - self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("Invalid user type", channel.json_body["error"]) - - -class UsersListTestCase(unittest.HomeserverTestCase): - - servlets = [ - synapse.rest.admin.register_servlets, - login.register_servlets, - ] - url = "/_synapse/admin/v2/users" - - def prepare(self, reactor, clock, hs): - self.admin_user = self.register_user("admin", "pass", admin=True) - self.admin_user_tok = self.login("admin", "pass") - - self.register_user("user1", "pass1", admin=False) - self.register_user("user2", "pass2", admin=False) - - def test_no_auth(self): - """ - Try to list users without authentication. - """ - request, channel = self.make_request("GET", self.url, b"{}") - self.render(request) - - self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual("M_MISSING_TOKEN", channel.json_body["errcode"]) - - def test_all_users(self): - """ - List all users, including deactivated users. - """ - request, channel = self.make_request( - "GET", - self.url + "?deactivated=true", - b"{}", - access_token=self.admin_user_tok, - ) - self.render(request) - - self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) - self.assertEqual(3, len(channel.json_body["users"])) - - class ShutdownRoomTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets_for_client_rest_resource, diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py new file mode 100644 index 0000000000..7352d609e6 --- /dev/null +++ b/tests/rest/admin/test_user.py @@ -0,0 +1,465 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import hmac +import json + +from mock import Mock + +import synapse.rest.admin +from synapse.api.constants import UserTypes +from synapse.rest.client.v1 import login + +from tests import unittest + + +class UserRegisterTestCase(unittest.HomeserverTestCase): + + servlets = [synapse.rest.admin.register_servlets_for_client_rest_resource] + + def make_homeserver(self, reactor, clock): + + self.url = "/_matrix/client/r0/admin/register" + + self.registration_handler = Mock() + self.identity_handler = Mock() + self.login_handler = Mock() + self.device_handler = Mock() + self.device_handler.check_device_registered = Mock(return_value="FAKE") + + self.datastore = Mock(return_value=Mock()) + self.datastore.get_current_state_deltas = Mock(return_value=(0, [])) + + self.secrets = Mock() + + self.hs = self.setup_test_homeserver() + + self.hs.config.registration_shared_secret = "shared" + + self.hs.get_media_repository = Mock() + self.hs.get_deactivate_account_handler = Mock() + + return self.hs + + def test_disabled(self): + """ + If there is no shared secret, registration through this method will be + prevented. + """ + self.hs.config.registration_shared_secret = None + + request, channel = self.make_request("POST", self.url, b"{}") + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual( + "Shared secret registration is not enabled", channel.json_body["error"] + ) + + def test_get_nonce(self): + """ + Calling GET on the endpoint will return a randomised nonce, using the + homeserver's secrets provider. + """ + secrets = Mock() + secrets.token_hex = Mock(return_value="abcd") + + self.hs.get_secrets = Mock(return_value=secrets) + + request, channel = self.make_request("GET", self.url) + self.render(request) + + self.assertEqual(channel.json_body, {"nonce": "abcd"}) + + def test_expired_nonce(self): + """ + Calling GET on the endpoint will return a randomised nonce, which will + only last for SALT_TIMEOUT (60s). + """ + request, channel = self.make_request("GET", self.url) + self.render(request) + nonce = channel.json_body["nonce"] + + # 59 seconds + self.reactor.advance(59) + + body = json.dumps({"nonce": nonce}) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("username must be specified", channel.json_body["error"]) + + # 61 seconds + self.reactor.advance(2) + + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("unrecognised nonce", channel.json_body["error"]) + + def test_register_incorrect_nonce(self): + """ + Only the provided nonce can be used, as it's checked in the MAC. + """ + request, channel = self.make_request("GET", self.url) + self.render(request) + nonce = channel.json_body["nonce"] + + want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) + want_mac.update(b"notthenonce\x00bob\x00abc123\x00admin") + want_mac = want_mac.hexdigest() + + body = json.dumps( + { + "nonce": nonce, + "username": "bob", + "password": "abc123", + "admin": True, + "mac": want_mac, + } + ) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("HMAC incorrect", channel.json_body["error"]) + + def test_register_correct_nonce(self): + """ + When the correct nonce is provided, and the right key is provided, the + user is registered. + """ + request, channel = self.make_request("GET", self.url) + self.render(request) + nonce = channel.json_body["nonce"] + + want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) + want_mac.update( + nonce.encode("ascii") + b"\x00bob\x00abc123\x00admin\x00support" + ) + want_mac = want_mac.hexdigest() + + body = json.dumps( + { + "nonce": nonce, + "username": "bob", + "password": "abc123", + "admin": True, + "user_type": UserTypes.SUPPORT, + "mac": want_mac, + } + ) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("@bob:test", channel.json_body["user_id"]) + + def test_nonce_reuse(self): + """ + A valid unrecognised nonce. + """ + request, channel = self.make_request("GET", self.url) + self.render(request) + nonce = channel.json_body["nonce"] + + want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) + want_mac.update(nonce.encode("ascii") + b"\x00bob\x00abc123\x00admin") + want_mac = want_mac.hexdigest() + + body = json.dumps( + { + "nonce": nonce, + "username": "bob", + "password": "abc123", + "admin": True, + "mac": want_mac, + } + ) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("@bob:test", channel.json_body["user_id"]) + + # Now, try and reuse it + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("unrecognised nonce", channel.json_body["error"]) + + def test_missing_parts(self): + """ + Synapse will complain if you don't give nonce, username, password, and + mac. Admin and user_types are optional. Additional checks are done for length + and type. + """ + + def nonce(): + request, channel = self.make_request("GET", self.url) + self.render(request) + return channel.json_body["nonce"] + + # + # Nonce check + # + + # Must be present + body = json.dumps({}) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("nonce must be specified", channel.json_body["error"]) + + # + # Username checks + # + + # Must be present + body = json.dumps({"nonce": nonce()}) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("username must be specified", channel.json_body["error"]) + + # Must be a string + body = json.dumps({"nonce": nonce(), "username": 1234}) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("Invalid username", channel.json_body["error"]) + + # Must not have null bytes + body = json.dumps({"nonce": nonce(), "username": "abcd\u0000"}) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("Invalid username", channel.json_body["error"]) + + # Must not have null bytes + body = json.dumps({"nonce": nonce(), "username": "a" * 1000}) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("Invalid username", channel.json_body["error"]) + + # + # Password checks + # + + # Must be present + body = json.dumps({"nonce": nonce(), "username": "a"}) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("password must be specified", channel.json_body["error"]) + + # Must be a string + body = json.dumps({"nonce": nonce(), "username": "a", "password": 1234}) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("Invalid password", channel.json_body["error"]) + + # Must not have null bytes + body = json.dumps({"nonce": nonce(), "username": "a", "password": "abcd\u0000"}) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("Invalid password", channel.json_body["error"]) + + # Super long + body = json.dumps({"nonce": nonce(), "username": "a", "password": "A" * 1000}) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("Invalid password", channel.json_body["error"]) + + # + # user_type check + # + + # Invalid user_type + body = json.dumps( + { + "nonce": nonce(), + "username": "a", + "password": "1234", + "user_type": "invalid", + } + ) + request, channel = self.make_request("POST", self.url, body.encode("utf8")) + self.render(request) + + self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("Invalid user type", channel.json_body["error"]) + + +class UsersListTestCase(unittest.HomeserverTestCase): + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + ] + url = "/_synapse/admin/v2/users" + + def prepare(self, reactor, clock, hs): + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.register_user("user1", "pass1", admin=False) + self.register_user("user2", "pass2", admin=False) + + def test_no_auth(self): + """ + Try to list users without authentication. + """ + request, channel = self.make_request("GET", self.url, b"{}") + self.render(request) + + self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("M_MISSING_TOKEN", channel.json_body["errcode"]) + + def test_all_users(self): + """ + List all users, including deactivated users. + """ + request, channel = self.make_request( + "GET", + self.url + "?deactivated=true", + b"{}", + access_token=self.admin_user_tok, + ) + self.render(request) + + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual(3, len(channel.json_body["users"])) + + +class UserRestTestCase(unittest.HomeserverTestCase): + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor, clock, hs): + self.store = hs.get_datastore() + + self.url = "/_synapse/admin/v2/users/@bob:test" + + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.other_user = self.register_user("user", "pass") + self.other_user_token = self.login("user", "pass") + + def test_requester_is_no_admin(self): + """ + If the user is not a server admin, an error is returned. + """ + self.hs.config.registration_shared_secret = None + + request, channel = self.make_request( + "GET", self.url, access_token=self.other_user_token, + ) + self.render(request) + + self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("You are not a server admin", channel.json_body["error"]) + + request, channel = self.make_request( + "PUT", self.url, access_token=self.other_user_token, content=b"{}", + ) + self.render(request) + + self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("You are not a server admin", channel.json_body["error"]) + + def test_requester_is_admin(self): + """ + If the user is a server admin, a new user is created. + """ + self.hs.config.registration_shared_secret = None + + body = json.dumps({"password": "abc123", "admin": True}) + + # Create user + request, channel = self.make_request( + "PUT", + self.url, + access_token=self.admin_user_tok, + content=body.encode(encoding="utf_8"), + ) + self.render(request) + + self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("@bob:test", channel.json_body["name"]) + self.assertEqual("bob", channel.json_body["displayname"]) + + # Get user + request, channel = self.make_request( + "GET", self.url, access_token=self.admin_user_tok, + ) + self.render(request) + + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("@bob:test", channel.json_body["name"]) + self.assertEqual("bob", channel.json_body["displayname"]) + self.assertEqual(1, channel.json_body["admin"]) + self.assertEqual(0, channel.json_body["is_guest"]) + self.assertEqual(0, channel.json_body["deactivated"]) + + # Modify user + body = json.dumps({"displayname": "foobar", "deactivated": True}) + + request, channel = self.make_request( + "PUT", + self.url, + access_token=self.admin_user_tok, + content=body.encode(encoding="utf_8"), + ) + self.render(request) + + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("@bob:test", channel.json_body["name"]) + self.assertEqual("foobar", channel.json_body["displayname"]) + self.assertEqual(True, channel.json_body["deactivated"]) + + # Get user + request, channel = self.make_request( + "GET", self.url, access_token=self.admin_user_tok, + ) + self.render(request) + + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual("@bob:test", channel.json_body["name"]) + self.assertEqual("foobar", channel.json_body["displayname"]) + self.assertEqual(1, channel.json_body["admin"]) + self.assertEqual(0, channel.json_body["is_guest"]) + self.assertEqual(1, channel.json_body["deactivated"]) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index ed5786865a..71a40a0a49 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -43,12 +43,14 @@ class RegistrationStoreTestCase(unittest.TestCase): # TODO(paul): Surely this field should be 'user_id', not 'name' "name": self.user_id, "password_hash": self.pwhash, + "admin": 0, "is_guest": 0, "consent_version": None, "consent_server_notice_sent": None, "appservice_id": None, "creation_ts": 1000, "user_type": None, + "deactivated": 0, }, (yield self.store.get_user_by_id(self.user_id)), ) -- cgit 1.5.1 From c2ba994dbb3f084c2c0cc8492e8a9456ec0f3747 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jan 2020 17:13:36 +0000 Subject: Add note about log_file no longer be accepted (#6674) --- CHANGES.md | 3 +++ UPGRADE.rst | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 690cbdaae9..d06b8c8ad3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ Synapse 1.8.0 (2020-01-09) ========================== +**WARNING**: As of this release Synapse will refuse to start if the `log_file` config option is specified. Support for the option was removed in v1.3.0. + + Bugfixes -------- diff --git a/UPGRADE.rst b/UPGRADE.rst index d9020f2663..a0202932b1 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -75,6 +75,15 @@ for example: wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb + +Upgrading to v1.8.0 +=================== + +Specifying a ``log_file`` config option will now cause Synapse to refuse to +start, and should be replaced by with the ``log_config`` option. Support for +the ``log_file`` option was removed in v1.3.0 and has since had no effect. + + Upgrading to v1.7.0 =================== -- cgit 1.5.1 From e97d1cf0014668b9d4883d4175b783088444b24b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 9 Jan 2020 17:21:30 +0000 Subject: Modify check_database to take a connection rather than a cursor We might not need the cursor at all. --- scripts/synapse_port_db | 25 +++++++------------------ synapse/storage/data_stores/__init__.py | 2 +- synapse/storage/engines/postgres.py | 17 +++++++++-------- synapse/storage/engines/sqlite.py | 2 +- 4 files changed, 18 insertions(+), 28 deletions(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index cb77314f1e..a3dafaffc9 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -447,15 +447,6 @@ class Porter(object): else: return - def setup_db(self, db_config: DatabaseConnectionConfig, engine): - db_conn = make_conn(db_config, engine) - prepare_database(db_conn, engine, config=None) - - db_conn.commit() - - return db_conn - - @defer.inlineCallbacks def build_db_store(self, db_config: DatabaseConnectionConfig): """Builds and returns a database store using the provided configuration. @@ -468,16 +459,14 @@ class Porter(object): self.progress.set_state("Preparing %s" % db_config.config["name"]) engine = create_engine(db_config.config) - conn = self.setup_db(db_config, engine) hs = MockHomeserver(self.hs_config) - store = Store(Database(hs, db_config, engine), conn, hs) - - yield store.db.runInteraction( - "%s_engine.check_database" % db_config.config["name"], - engine.check_database, - ) + with make_conn(db_config, engine) as db_conn: + engine.check_database(db_conn) + prepare_database(db_conn, engine, config=None) + store = Store(Database(hs, db_config, engine), db_conn, hs) + db_conn.commit() return store @@ -502,7 +491,7 @@ class Porter(object): @defer.inlineCallbacks def run(self): try: - self.sqlite_store = yield self.build_db_store( + self.sqlite_store = self.build_db_store( DatabaseConnectionConfig("master-sqlite", self.sqlite_config) ) @@ -518,7 +507,7 @@ class Porter(object): ) defer.returnValue(None) - self.postgres_store = yield self.build_db_store( + self.postgres_store = self.build_db_store( self.hs_config.get_single_database() ) diff --git a/synapse/storage/data_stores/__init__.py b/synapse/storage/data_stores/__init__.py index 092e803799..e1d03429ca 100644 --- a/synapse/storage/data_stores/__init__.py +++ b/synapse/storage/data_stores/__init__.py @@ -47,7 +47,7 @@ class DataStores(object): with make_conn(database_config, engine) as db_conn: logger.info("Preparing database %r...", db_name) - engine.check_database(db_conn.cursor()) + engine.check_database(db_conn) prepare_database( db_conn, engine, hs.config, data_stores=database_config.data_stores, ) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index b7c4eda338..ba19785fd7 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -32,14 +32,15 @@ class PostgresEngine(object): self.synchronous_commit = database_config.get("synchronous_commit", True) self._version = None # unknown as yet - def check_database(self, txn): - txn.execute("SHOW SERVER_ENCODING") - rows = txn.fetchall() - if rows and rows[0][0] != "UTF8": - raise IncorrectDatabaseSetup( - "Database has incorrect encoding: '%s' instead of 'UTF8'\n" - "See docs/postgres.rst for more information." % (rows[0][0],) - ) + def check_database(self, db_conn): + with db_conn.cursor() as txn: + txn.execute("SHOW SERVER_ENCODING") + rows = txn.fetchall() + if rows and rows[0][0] != "UTF8": + raise IncorrectDatabaseSetup( + "Database has incorrect encoding: '%s' instead of 'UTF8'\n" + "See docs/postgres.rst for more information." % (rows[0][0],) + ) def convert_param_style(self, sql): return sql.replace("?", "%s") diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index df039a072d..3b3c13360b 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -53,7 +53,7 @@ class Sqlite3Engine(object): """ return False - def check_database(self, txn): + def check_database(self, db_conn): pass def convert_param_style(self, sql): -- cgit 1.5.1 From e48ba84e0bfe081814941b74e610ddcd168a3ce8 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 9 Jan 2020 17:33:41 +0000 Subject: Check postgres version in check_database this saves doing it on each connection, and will allow us to pass extra options in. --- synapse/storage/engines/postgres.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index ba19785fd7..2a285e018c 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -33,6 +33,16 @@ class PostgresEngine(object): self._version = None # unknown as yet def check_database(self, db_conn): + # Get the version of PostgreSQL that we're using. As per the psycopg2 + # docs: The number is formed by converting the major, minor, and + # revision numbers into two-decimal-digit numbers and appending them + # together. For example, version 8.1.5 will be returned as 80105 + self._version = db_conn.server_version + + # Are we on a supported PostgreSQL version? + if self._version < 90500: + raise RuntimeError("Synapse requires PostgreSQL 9.5+ or above.") + with db_conn.cursor() as txn: txn.execute("SHOW SERVER_ENCODING") rows = txn.fetchall() @@ -46,17 +56,6 @@ class PostgresEngine(object): return sql.replace("?", "%s") def on_new_connection(self, db_conn): - - # Get the version of PostgreSQL that we're using. As per the psycopg2 - # docs: The number is formed by converting the major, minor, and - # revision numbers into two-decimal-digit numbers and appending them - # together. For example, version 8.1.5 will be returned as 80105 - self._version = db_conn.server_version - - # Are we on a supported PostgreSQL version? - if self._version < 90500: - raise RuntimeError("Synapse requires PostgreSQL 9.5+ or above.") - db_conn.set_isolation_level( self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ ) @@ -120,8 +119,8 @@ class PostgresEngine(object): Returns: string """ - # note that this is a bit of a hack because it relies on on_new_connection - # having been called at least once. Still, that should be a safe bet here. + # note that this is a bit of a hack because it relies on check_database + # having been called. Still, that should be a safe bet here. numver = self._version assert numver is not None -- cgit 1.5.1 From bf468211805900e767b6b07a2bfa6046f70efb7a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 9 Jan 2020 17:46:52 +0000 Subject: Refuse to start if sqlite is older than 3.11.0 --- scripts/synapse_port_db | 16 ++++++++++++---- synapse/storage/engines/postgres.py | 4 ++-- synapse/storage/engines/sqlite.py | 7 +++++-- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index a3dafaffc9..f135c8bc54 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -447,11 +447,15 @@ class Porter(object): else: return - def build_db_store(self, db_config: DatabaseConnectionConfig): + def build_db_store( + self, db_config: DatabaseConnectionConfig, allow_outdated_version: bool = False, + ): """Builds and returns a database store using the provided configuration. Args: - config: The database configuration + db_config: The database configuration + allow_outdated_version: True to suppress errors about the database server + version being too old to run a complete synapse Returns: The built Store object. @@ -463,7 +467,9 @@ class Porter(object): hs = MockHomeserver(self.hs_config) with make_conn(db_config, engine) as db_conn: - engine.check_database(db_conn) + engine.check_database( + db_conn, allow_outdated_version=allow_outdated_version + ) prepare_database(db_conn, engine, config=None) store = Store(Database(hs, db_config, engine), db_conn, hs) db_conn.commit() @@ -491,8 +497,10 @@ class Porter(object): @defer.inlineCallbacks def run(self): try: + # we allow people to port away from outdated versions of sqlite. self.sqlite_store = self.build_db_store( - DatabaseConnectionConfig("master-sqlite", self.sqlite_config) + DatabaseConnectionConfig("master-sqlite", self.sqlite_config), + allow_outdated_version=True, ) # Check if all background updates are done, abort if not. diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 2a285e018c..c84cb452b0 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -32,7 +32,7 @@ class PostgresEngine(object): self.synchronous_commit = database_config.get("synchronous_commit", True) self._version = None # unknown as yet - def check_database(self, db_conn): + def check_database(self, db_conn, allow_outdated_version: bool = False): # Get the version of PostgreSQL that we're using. As per the psycopg2 # docs: The number is formed by converting the major, minor, and # revision numbers into two-decimal-digit numbers and appending them @@ -40,7 +40,7 @@ class PostgresEngine(object): self._version = db_conn.server_version # Are we on a supported PostgreSQL version? - if self._version < 90500: + if not allow_outdated_version and self._version < 90500: raise RuntimeError("Synapse requires PostgreSQL 9.5+ or above.") with db_conn.cursor() as txn: diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 3b3c13360b..cbf52f5191 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -53,8 +53,11 @@ class Sqlite3Engine(object): """ return False - def check_database(self, db_conn): - pass + def check_database(self, db_conn, allow_outdated_version: bool = False): + if not allow_outdated_version: + version = self.module.sqlite_version_info + if version < (3, 11, 0): + raise RuntimeError("Synapse requires sqlite 3.11 or above.") def convert_param_style(self, sql): return sql -- cgit 1.5.1 From c3843fd075c5c20f800837afce534de352517db6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 9 Jan 2020 17:52:12 +0000 Subject: changelog --- changelog.d/6675.removal | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/6675.removal diff --git a/changelog.d/6675.removal b/changelog.d/6675.removal new file mode 100644 index 0000000000..95df9a2d83 --- /dev/null +++ b/changelog.d/6675.removal @@ -0,0 +1 @@ +Synapse no longer supports versions of SQLite before 3.11, and will refuse to start when configured to use an older version. Administrators are recommended to migrate their database to Postgres (see instructions [here](docs/postgres.md)). -- cgit 1.5.1 From 937dea42e72c982ab532a2b558f0540e5d5f4f67 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 9 Jan 2020 18:01:08 +0000 Subject: update install notes for CentOS --- INSTALL.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 9da2e3c734..d25fcf0753 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -133,6 +133,11 @@ sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \ sudo yum groupinstall "Development Tools" ``` +Note that Synapse does not support versions of SQLite before 3.11, and CentOS 7 +uses SQLite 3.7. You may be able to work around this by installing a more +recent SQLite version, but it is recommended that you instead use a Postgres +database: see [docs/postgres.md](docs/postgres.md). + #### macOS Installing prerequisites on macOS: -- cgit 1.5.1 From 47e63cc67a059fdc28f01297740b1071c0c6ab5c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sat, 11 Jan 2020 11:48:43 +0000 Subject: Pass the module_api into the SamlMappingProvider ... for consistency with other modules, and because we'll need it sooner or later and it will be a pain to introduce later. --- synapse/handlers/saml_handler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml_handler.py index 0082f85c26..96f3d016fe 100644 --- a/synapse/handlers/saml_handler.py +++ b/synapse/handlers/saml_handler.py @@ -24,6 +24,7 @@ from saml2.client import Saml2Client from synapse.api.errors import SynapseError from synapse.config import ConfigError from synapse.http.servlet import parse_string +from synapse.module_api import ModuleApi from synapse.rest.client.v1.login import SSOAuthHandler from synapse.types import ( UserID, @@ -59,7 +60,8 @@ class SamlHandler: # plugin to do custom mapping from saml response to mxid self._user_mapping_provider = hs.config.saml2_user_mapping_provider_class( - hs.config.saml2_user_mapping_provider_config + hs.config.saml2_user_mapping_provider_config, + ModuleApi(hs, hs.get_auth_handler()), ) # identifier for the external_ids table @@ -265,11 +267,12 @@ class SamlConfig(object): class DefaultSamlMappingProvider(object): __version__ = "0.0.1" - def __init__(self, parsed_config: SamlConfig): + def __init__(self, parsed_config: SamlConfig, module_api: ModuleApi): """The default SAML user mapping provider Args: parsed_config: Module configuration + module_api: module api proxy """ self._mxid_source_attribute = parsed_config.mxid_source_attribute self._mxid_mapper = parsed_config.mxid_mapper -- cgit 1.5.1 From dc69a1cf432e8be2d37220e641345a70f3c692fc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sat, 11 Jan 2020 01:01:53 +0000 Subject: Pass client redirect URL into SAML mapping providers --- synapse/handlers/saml_handler.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml_handler.py index 96f3d016fe..107f97032b 100644 --- a/synapse/handlers/saml_handler.py +++ b/synapse/handlers/saml_handler.py @@ -114,10 +114,10 @@ class SamlHandler: # the dict. self.expire_sessions() - user_id = await self._map_saml_response_to_user(resp_bytes) + user_id = await self._map_saml_response_to_user(resp_bytes, relay_state) self._sso_auth_handler.complete_sso_login(user_id, request, relay_state) - async def _map_saml_response_to_user(self, resp_bytes): + async def _map_saml_response_to_user(self, resp_bytes, client_redirect_url): try: saml2_auth = self._saml_client.parse_authn_request_response( resp_bytes, @@ -185,7 +185,7 @@ class SamlHandler: # Map saml response to user attributes using the configured mapping provider for i in range(1000): attribute_dict = self._user_mapping_provider.saml_response_to_user_attributes( - saml2_auth, i + saml2_auth, i, client_redirect_url=client_redirect_url, ) logger.debug( @@ -218,6 +218,8 @@ class SamlHandler: 500, "Unable to generate a Matrix ID from the SAML response" ) + logger.info("Mapped SAML user to local part %s", localpart) + registered_user_id = await self._registration_handler.register_user( localpart=localpart, default_display_name=displayname ) @@ -278,7 +280,10 @@ class DefaultSamlMappingProvider(object): self._mxid_mapper = parsed_config.mxid_mapper def saml_response_to_user_attributes( - self, saml_response: saml2.response.AuthnResponse, failures: int = 0, + self, + saml_response: saml2.response.AuthnResponse, + failures: int, + client_redirect_url: str, ) -> dict: """Maps some text from a SAML response to attributes of a new user @@ -288,6 +293,8 @@ class DefaultSamlMappingProvider(object): failures: How many times a call to this function with this saml_response has resulted in a failure + client_redirect_url: where the client wants to redirect to + Returns: dict: A dict containing new user attributes. Possible keys: * mxid_localpart (str): Required. The localpart of the user's mxid -- cgit 1.5.1 From d56e95ea8ba316f1fddfe33fa080d5fd0b19c008 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 12 Jan 2020 21:42:15 +0000 Subject: changelog --- changelog.d/6689.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/6689.misc diff --git a/changelog.d/6689.misc b/changelog.d/6689.misc new file mode 100644 index 0000000000..17f15e73a8 --- /dev/null +++ b/changelog.d/6689.misc @@ -0,0 +1 @@ +Updates to the SAML mapping provider API. -- cgit 1.5.1 From da4e52544e326b707af6168a63d65eace34d8e9c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sat, 11 Jan 2020 13:00:24 +0000 Subject: comment for run_in_background --- synapse/logging/context.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/logging/context.py b/synapse/logging/context.py index 33b322209d..1b940842f6 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -571,6 +571,9 @@ def run_in_background(f, *args, **kwargs): yield or await on (for instance because you want to pass it to deferred.gatherResults()). + If f returns a Coroutine object, it will be wrapped into a Deferred (which will have + the side effect of executing the coroutine). + Note that if you completely discard the result, you should make sure that `f` doesn't raise any deferred exceptions, otherwise a scary-looking CRITICAL error about an unhandled error will be logged without much -- cgit 1.5.1 From feee8199734c9d8a18fa0be12fc5ec09ae140a3a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 13 Jan 2020 12:41:51 +0000 Subject: Fix exceptions on requests for non-ascii urls (#6682) Fixes #6402 --- changelog.d/6682.bugfix | 2 ++ synapse/http/site.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/6682.bugfix diff --git a/changelog.d/6682.bugfix b/changelog.d/6682.bugfix new file mode 100644 index 0000000000..d48ea31477 --- /dev/null +++ b/changelog.d/6682.bugfix @@ -0,0 +1,2 @@ +Fix "CRITICAL" errors being logged when a request is received for a uri containing non-ascii characters. + diff --git a/synapse/http/site.py b/synapse/http/site.py index 9f2d035fa0..911251c0bc 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -88,7 +88,7 @@ class SynapseRequest(Request): def get_redacted_uri(self): uri = self.uri if isinstance(uri, bytes): - uri = self.uri.decode("ascii") + uri = self.uri.decode("ascii", errors="replace") return redact_uri(uri) def get_method(self): -- cgit 1.5.1 From 8039685051c08354b64890abb2522f2535c784b8 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 13 Jan 2020 12:42:44 +0000 Subject: Allow additional_resources to implement Resource directly (#6686) AdditionalResource really doesn't add any value, and it gets in the way for resources which want to support child resources or the like. So, if the resource object already implements the IResource interface, don't bother wrapping it. --- changelog.d/6686.misc | 1 + synapse/app/homeserver.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 changelog.d/6686.misc diff --git a/changelog.d/6686.misc b/changelog.d/6686.misc new file mode 100644 index 0000000000..4070f2e563 --- /dev/null +++ b/changelog.d/6686.misc @@ -0,0 +1 @@ +Allow additional_resources to implement IResource directly. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index e5b44a5eed..c2a334a2b0 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -31,7 +31,7 @@ from prometheus_client import Gauge from twisted.application import service from twisted.internet import defer, reactor from twisted.python.failure import Failure -from twisted.web.resource import EncodingResourceWrapper, NoResource +from twisted.web.resource import EncodingResourceWrapper, IResource, NoResource from twisted.web.server import GzipEncoderFactory from twisted.web.static import File @@ -109,7 +109,16 @@ class SynapseHomeServer(HomeServer): for path, resmodule in additional_resources.items(): handler_cls, config = load_module(resmodule) handler = handler_cls(config, module_api) - resources[path] = AdditionalResource(self, handler.handle_request) + if IResource.providedBy(handler): + resource = handler + elif hasattr(handler, "handle_request"): + resource = AdditionalResource(self, handler.handle_request) + else: + raise ConfigError( + "additional_resource %s does not implement a known interface" + % (resmodule["module"],) + ) + resources[path] = resource # try to find something useful to redirect '/' to if WEB_CLIENT_PREFIX in resources: -- cgit 1.5.1 From 2d07c73777e837213f1c3c85b9cb446aac8b6170 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 13 Jan 2020 12:47:30 +0000 Subject: Don't assign numeric IDs for empty usernames (#6690) Fix a bug where we would assign a numeric userid if somebody tried registering with an empty username --- changelog.d/6690.bugfix | 1 + synapse/handlers/register.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/6690.bugfix diff --git a/changelog.d/6690.bugfix b/changelog.d/6690.bugfix new file mode 100644 index 0000000000..30ce1dc9f7 --- /dev/null +++ b/changelog.d/6690.bugfix @@ -0,0 +1 @@ +Fix a bug where we would assign a numeric userid if somebody tried registering with an empty username. diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 8a7d965feb..885da82985 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -174,7 +174,7 @@ class RegistrationHandler(BaseHandler): if password: password_hash = yield self._auth_handler.hash(password) - if localpart: + if localpart is not None: yield self.check_username(localpart, guest_access_token=guest_access_token) was_guest = guest_access_token is not None -- cgit 1.5.1 From 326c893d24e0f03b62bd9d1136a335f329bb8528 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 13 Jan 2020 12:48:22 +0000 Subject: Kill off RegistrationError (#6691) This is pretty pointless. Let's just use SynapseError. --- changelog.d/6691.misc | 1 + synapse/api/errors.py | 6 ------ synapse/handlers/register.py | 12 +++--------- tests/handlers/test_register.py | 2 -- 4 files changed, 4 insertions(+), 17 deletions(-) create mode 100644 changelog.d/6691.misc diff --git a/changelog.d/6691.misc b/changelog.d/6691.misc new file mode 100644 index 0000000000..104e9ce648 --- /dev/null +++ b/changelog.d/6691.misc @@ -0,0 +1 @@ +Remove redundant RegistrationError class. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 5853a54c95..9e9844b47c 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -158,12 +158,6 @@ class UserDeactivatedError(SynapseError): ) -class RegistrationError(SynapseError): - """An error raised when a registration event fails.""" - - pass - - class FederationDeniedError(SynapseError): """An error raised when the server tries to federate with a server which is not on its federation whitelist. diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 885da82985..7ffc194f0c 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -20,13 +20,7 @@ from twisted.internet import defer from synapse import types from synapse.api.constants import MAX_USERID_LENGTH, LoginType -from synapse.api.errors import ( - AuthError, - Codes, - ConsentNotGivenError, - RegistrationError, - SynapseError, -) +from synapse.api.errors import AuthError, Codes, ConsentNotGivenError, SynapseError from synapse.config.server import is_threepid_reserved from synapse.http.servlet import assert_params_in_dict from synapse.replication.http.login import RegisterDeviceReplicationServlet @@ -165,7 +159,7 @@ class RegistrationHandler(BaseHandler): Returns: Deferred[str]: user_id Raises: - RegistrationError if there was a problem registering. + SynapseError if there was a problem registering. """ yield self.check_registration_ratelimit(address) @@ -182,7 +176,7 @@ class RegistrationHandler(BaseHandler): if not was_guest: try: int(localpart) - raise RegistrationError( + raise SynapseError( 400, "Numeric user IDs are reserved for guest users." ) except ValueError: diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 1e9ba3a201..e2915eb7b1 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -269,8 +269,6 @@ class RegistrationTestCase(unittest.HomeserverTestCase): one will be randomly generated. Returns: A tuple of (user_id, access_token). - Raises: - RegistrationError if there was a problem registering. """ if localpart is None: raise SynapseError(400, "Request must include user id") -- cgit 1.5.1 From 47f4f493f0886af5c9aad5c78885bb6869018dda Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 13 Jan 2020 15:32:02 +0000 Subject: Document more supported endpoints for workers (#6698) --- changelog.d/6698.doc | 1 + docs/workers.md | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 changelog.d/6698.doc diff --git a/changelog.d/6698.doc b/changelog.d/6698.doc new file mode 100644 index 0000000000..5aba51252d --- /dev/null +++ b/changelog.d/6698.doc @@ -0,0 +1 @@ +Add more endpoints to the documentation for Synapse workers. diff --git a/docs/workers.md b/docs/workers.md index 1b5d94f5eb..f4283aeb05 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -168,8 +168,11 @@ endpoints matching the following regular expressions: ^/_matrix/federation/v1/make_join/ ^/_matrix/federation/v1/make_leave/ ^/_matrix/federation/v1/send_join/ + ^/_matrix/federation/v2/send_join/ ^/_matrix/federation/v1/send_leave/ + ^/_matrix/federation/v2/send_leave/ ^/_matrix/federation/v1/invite/ + ^/_matrix/federation/v2/invite/ ^/_matrix/federation/v1/query_auth/ ^/_matrix/federation/v1/event_auth/ ^/_matrix/federation/v1/exchange_third_party_invite/ @@ -288,6 +291,7 @@ file. For example: Handles some event creation. It can handle REST endpoints matching: ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send + ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/ ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$ ^/_matrix/client/(api/v1|r0|unstable)/join/ ^/_matrix/client/(api/v1|r0|unstable)/profile/ -- cgit 1.5.1 From 1177d3f3a33bd3ae1eef46fba360d319598359ad Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 13 Jan 2020 18:10:43 +0000 Subject: Quarantine media by ID or user ID (#6681) --- changelog.d/6681.feature | 1 + docs/admin_api/media_admin_api.md | 76 ++++++- docs/workers.md | 4 +- synapse/rest/admin/media.py | 68 +++++- synapse/storage/data_stores/main/room.py | 116 ++++++++++- tests/rest/admin/test_admin.py | 341 +++++++++++++++++++++++++++++++ tests/rest/client/v1/utils.py | 37 ++++ 7 files changed, 632 insertions(+), 11 deletions(-) create mode 100644 changelog.d/6681.feature diff --git a/changelog.d/6681.feature b/changelog.d/6681.feature new file mode 100644 index 0000000000..5cf19a4e0e --- /dev/null +++ b/changelog.d/6681.feature @@ -0,0 +1 @@ +Add new quarantine media admin APIs to quarantine by media ID or by user who uploaded the media. diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 8b3666d5f5..46ba7a1a71 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -22,19 +22,81 @@ It returns a JSON body like the following: } ``` -# Quarantine media in a room +# Quarantine media -This API 'quarantines' all the media in a room. +Quarantining media means that it is marked as inaccessible by users. It applies +to any local media, and any locally-cached copies of remote media. -The API is: +The media file itself (and any thumbnails) is not deleted from the server. + +## Quarantining media by ID + +This API quarantines a single piece of local or remote media. + +Request: ``` -POST /_synapse/admin/v1/quarantine_media/ +POST /_synapse/admin/v1/media/quarantine// {} ``` -Quarantining media means that it is marked as inaccessible by users. It applies -to any local media, and any locally-cached copies of remote media. +Where `server_name` is in the form of `example.org`, and `media_id` is in the +form of `abcdefg12345...`. + +Response: + +``` +{} +``` + +## Quarantining media in a room + +This API quarantines all local and remote media in a room. + +Request: + +``` +POST /_synapse/admin/v1/room//media/quarantine + +{} +``` + +Where `room_id` is in the form of `!roomid12345:example.org`. + +Response: + +``` +{ + "num_quarantined": 10 # The number of media items successfully quarantined +} +``` + +Note that there is a legacy endpoint, `POST +/_synapse/admin/v1/quarantine_media/`, that operates the same. +However, it is deprecated and may be removed in a future release. + +## Quarantining all media of a user + +This API quarantines all *local* media that a *local* user has uploaded. That is to say, if +you would like to quarantine media uploaded by a user on a remote homeserver, you should +instead use one of the other APIs. + +Request: + +``` +POST /_synapse/admin/v1/user//media/quarantine + +{} +``` + +Where `user_id` is in the form of `@bob:example.org`. + +Response: + +``` +{ + "num_quarantined": 10 # The number of media items successfully quarantined +} +``` -The media file itself (and any thumbnails) is not deleted from the server. diff --git a/docs/workers.md b/docs/workers.md index f4283aeb05..0ab269fd96 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -202,7 +202,9 @@ Handles the media repository. It can handle all endpoints starting with: ... and the following regular expressions matching media-specific administration APIs: ^/_synapse/admin/v1/purge_media_cache$ - ^/_synapse/admin/v1/room/.*/media$ + ^/_synapse/admin/v1/room/.*/media.*$ + ^/_synapse/admin/v1/user/.*/media.*$ + ^/_synapse/admin/v1/media/.*$ ^/_synapse/admin/v1/quarantine_media/.*$ You should also set `enable_media_repo: False` in the shared configuration diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index fa833e54cf..3a445d6eed 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -32,16 +32,24 @@ class QuarantineMediaInRoom(RestServlet): this server. """ - PATTERNS = historical_admin_path_patterns("/quarantine_media/(?P[^/]+)") + PATTERNS = ( + historical_admin_path_patterns("/room/(?P[^/]+)/media/quarantine") + + + # This path kept around for legacy reasons + historical_admin_path_patterns("/quarantine_media/(?P![^/]+)") + ) def __init__(self, hs): self.store = hs.get_datastore() self.auth = hs.get_auth() - async def on_POST(self, request, room_id): + async def on_POST(self, request, room_id: str): requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester.user) + logging.info("Quarantining room: %s", room_id) + + # Quarantine all media in this room num_quarantined = await self.store.quarantine_media_ids_in_room( room_id, requester.user.to_string() ) @@ -49,6 +57,60 @@ class QuarantineMediaInRoom(RestServlet): return 200, {"num_quarantined": num_quarantined} +class QuarantineMediaByUser(RestServlet): + """Quarantines all local media by a given user so that no one can download it via + this server. + """ + + PATTERNS = historical_admin_path_patterns( + "/user/(?P[^/]+)/media/quarantine" + ) + + def __init__(self, hs): + self.store = hs.get_datastore() + self.auth = hs.get_auth() + + async def on_POST(self, request, user_id: str): + requester = await self.auth.get_user_by_req(request) + await assert_user_is_admin(self.auth, requester.user) + + logging.info("Quarantining local media by user: %s", user_id) + + # Quarantine all media this user has uploaded + num_quarantined = await self.store.quarantine_media_ids_by_user( + user_id, requester.user.to_string() + ) + + return 200, {"num_quarantined": num_quarantined} + + +class QuarantineMediaByID(RestServlet): + """Quarantines local or remote media by a given ID so that no one can download + it via this server. + """ + + PATTERNS = historical_admin_path_patterns( + "/media/quarantine/(?P[^/]+)/(?P[^/]+)" + ) + + def __init__(self, hs): + self.store = hs.get_datastore() + self.auth = hs.get_auth() + + async def on_POST(self, request, server_name: str, media_id: str): + requester = await self.auth.get_user_by_req(request) + await assert_user_is_admin(self.auth, requester.user) + + logging.info("Quarantining local media by ID: %s/%s", server_name, media_id) + + # Quarantine this media id + await self.store.quarantine_media_by_id( + server_name, media_id, requester.user.to_string() + ) + + return 200, {} + + class ListMediaInRoom(RestServlet): """Lists all of the media in a given room. """ @@ -94,4 +156,6 @@ def register_servlets_for_media_repo(hs, http_server): """ PurgeMediaCacheRestServlet(hs).register(http_server) QuarantineMediaInRoom(hs).register(http_server) + QuarantineMediaByID(hs).register(http_server) + QuarantineMediaByUser(hs).register(http_server) ListMediaInRoom(hs).register(http_server) diff --git a/synapse/storage/data_stores/main/room.py b/synapse/storage/data_stores/main/room.py index 8636d75030..49bab62be3 100644 --- a/synapse/storage/data_stores/main/room.py +++ b/synapse/storage/data_stores/main/room.py @@ -18,7 +18,7 @@ import collections import logging import re from abc import abstractmethod -from typing import Optional, Tuple +from typing import List, Optional, Tuple from six import integer_types @@ -399,6 +399,8 @@ class RoomWorkerStore(SQLBaseStore): the associated media """ + logger.info("Quarantining media in room: %s", room_id) + def _quarantine_media_in_room_txn(txn): local_mxcs, remote_mxcs = self._get_media_mxcs_in_room_txn(txn, room_id) total_media_quarantined = 0 @@ -494,6 +496,118 @@ class RoomWorkerStore(SQLBaseStore): return local_media_mxcs, remote_media_mxcs + def quarantine_media_by_id( + self, server_name: str, media_id: str, quarantined_by: str, + ): + """quarantines a single local or remote media id + + Args: + server_name: The name of the server that holds this media + media_id: The ID of the media to be quarantined + quarantined_by: The user ID that initiated the quarantine request + """ + logger.info("Quarantining media: %s/%s", server_name, media_id) + is_local = server_name == self.config.server_name + + def _quarantine_media_by_id_txn(txn): + local_mxcs = [media_id] if is_local else [] + remote_mxcs = [(server_name, media_id)] if not is_local else [] + + return self._quarantine_media_txn( + txn, local_mxcs, remote_mxcs, quarantined_by + ) + + return self.db.runInteraction( + "quarantine_media_by_user", _quarantine_media_by_id_txn + ) + + def quarantine_media_ids_by_user(self, user_id: str, quarantined_by: str): + """quarantines all local media associated with a single user + + Args: + user_id: The ID of the user to quarantine media of + quarantined_by: The ID of the user who made the quarantine request + """ + + def _quarantine_media_by_user_txn(txn): + local_media_ids = self._get_media_ids_by_user_txn(txn, user_id) + return self._quarantine_media_txn(txn, local_media_ids, [], quarantined_by) + + return self.db.runInteraction( + "quarantine_media_by_user", _quarantine_media_by_user_txn + ) + + def _get_media_ids_by_user_txn(self, txn, user_id: str, filter_quarantined=True): + """Retrieves local media IDs by a given user + + Args: + txn (cursor) + user_id: The ID of the user to retrieve media IDs of + + Returns: + The local and remote media as a lists of tuples where the key is + the hostname and the value is the media ID. + """ + # Local media + sql = """ + SELECT media_id + FROM local_media_repository + WHERE user_id = ? + """ + if filter_quarantined: + sql += "AND quarantined_by IS NULL" + txn.execute(sql, (user_id,)) + + local_media_ids = [row[0] for row in txn] + + # TODO: Figure out all remote media a user has referenced in a message + + return local_media_ids + + def _quarantine_media_txn( + self, + txn, + local_mxcs: List[str], + remote_mxcs: List[Tuple[str, str]], + quarantined_by: str, + ) -> int: + """Quarantine local and remote media items + + Args: + txn (cursor) + local_mxcs: A list of local mxc URLs + remote_mxcs: A list of (remote server, media id) tuples representing + remote mxc URLs + quarantined_by: The ID of the user who initiated the quarantine request + Returns: + The total number of media items quarantined + """ + total_media_quarantined = 0 + + # Update all the tables to set the quarantined_by flag + txn.executemany( + """ + UPDATE local_media_repository + SET quarantined_by = ? + WHERE media_id = ? + """, + ((quarantined_by, media_id) for media_id in local_mxcs), + ) + + txn.executemany( + """ + UPDATE remote_media_cache + SET quarantined_by = ? + WHERE media_origin = ? AND media_id = ? + """, + ((quarantined_by, origin, media_id) for origin, media_id in remote_mxcs), + ) + + total_media_quarantined += len(local_mxcs) + total_media_quarantined += len(remote_mxcs) + + return total_media_quarantined + class RoomBackgroundUpdateStore(SQLBaseStore): REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory" diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 6ceb483aa8..7a7e898843 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -14,11 +14,17 @@ # limitations under the License. import json +import os +import urllib.parse +from binascii import unhexlify from mock import Mock +from twisted.internet.defer import Deferred + import synapse.rest.admin from synapse.http.server import JsonResource +from synapse.logging.context import make_deferred_yieldable from synapse.rest.admin import VersionServlet from synapse.rest.client.v1 import events, login, room from synapse.rest.client.v2_alpha import groups @@ -346,3 +352,338 @@ class PurgeRoomTestCase(unittest.HomeserverTestCase): self.assertEqual(count, 0, msg="Rows not purged in {}".format(table)) test_purge_room.skip = "Disabled because it's currently broken" + + +class QuarantineMediaTestCase(unittest.HomeserverTestCase): + """Test /quarantine_media admin API. + """ + + servlets = [ + synapse.rest.admin.register_servlets, + synapse.rest.admin.register_servlets_for_media_repo, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor, clock, hs): + self.store = hs.get_datastore() + self.hs = hs + + # Allow for uploading and downloading to/from the media repo + self.media_repo = hs.get_media_repository_resource() + self.download_resource = self.media_repo.children[b"download"] + self.upload_resource = self.media_repo.children[b"upload"] + self.image_data = unhexlify( + b"89504e470d0a1a0a0000000d4948445200000001000000010806" + b"0000001f15c4890000000a49444154789c63000100000500010d" + b"0a2db40000000049454e44ae426082" + ) + + def make_homeserver(self, reactor, clock): + + self.fetches = [] + + def get_file(destination, path, output_stream, args=None, max_size=None): + """ + Returns tuple[int,dict,str,int] of file length, response headers, + absolute URI, and response code. + """ + + def write_to(r): + data, response = r + output_stream.write(data) + return response + + d = Deferred() + d.addCallback(write_to) + self.fetches.append((d, destination, path, args)) + return make_deferred_yieldable(d) + + client = Mock() + client.get_file = get_file + + self.storage_path = self.mktemp() + self.media_store_path = self.mktemp() + os.mkdir(self.storage_path) + os.mkdir(self.media_store_path) + + config = self.default_config() + config["media_store_path"] = self.media_store_path + config["thumbnail_requirements"] = {} + config["max_image_pixels"] = 2000000 + + provider_config = { + "module": "synapse.rest.media.v1.storage_provider.FileStorageProviderBackend", + "store_local": True, + "store_synchronous": False, + "store_remote": True, + "config": {"directory": self.storage_path}, + } + config["media_storage_providers"] = [provider_config] + + hs = self.setup_test_homeserver(config=config, http_client=client) + + return hs + + def test_quarantine_media_requires_admin(self): + self.register_user("nonadmin", "pass", admin=False) + non_admin_user_tok = self.login("nonadmin", "pass") + + # Attempt quarantine media APIs as non-admin + url = "/_synapse/admin/v1/media/quarantine/example.org/abcde12345" + request, channel = self.make_request( + "POST", url.encode("ascii"), access_token=non_admin_user_tok, + ) + self.render(request) + + # Expect a forbidden error + self.assertEqual( + 403, + int(channel.result["code"]), + msg="Expected forbidden on quarantining media as a non-admin", + ) + + # And the roomID/userID endpoint + url = "/_synapse/admin/v1/room/!room%3Aexample.com/media/quarantine" + request, channel = self.make_request( + "POST", url.encode("ascii"), access_token=non_admin_user_tok, + ) + self.render(request) + + # Expect a forbidden error + self.assertEqual( + 403, + int(channel.result["code"]), + msg="Expected forbidden on quarantining media as a non-admin", + ) + + def test_quarantine_media_by_id(self): + self.register_user("id_admin", "pass", admin=True) + admin_user_tok = self.login("id_admin", "pass") + + self.register_user("id_nonadmin", "pass", admin=False) + non_admin_user_tok = self.login("id_nonadmin", "pass") + + # Upload some media into the room + response = self.helper.upload_media( + self.upload_resource, self.image_data, tok=admin_user_tok + ) + + # Extract media ID from the response + server_name_and_media_id = response["content_uri"][ + 6: + ] # Cut off the 'mxc://' bit + server_name, media_id = server_name_and_media_id.split("/") + + # Attempt to access the media + request, channel = self.make_request( + "GET", + server_name_and_media_id, + shorthand=False, + access_token=non_admin_user_tok, + ) + request.render(self.download_resource) + self.pump(1.0) + + # Should be successful + self.assertEqual(200, int(channel.code), msg=channel.result["body"]) + + # Quarantine the media + url = "/_synapse/admin/v1/media/quarantine/%s/%s" % ( + urllib.parse.quote(server_name), + urllib.parse.quote(media_id), + ) + request, channel = self.make_request("POST", url, access_token=admin_user_tok,) + self.render(request) + self.pump(1.0) + self.assertEqual(200, int(channel.code), msg=channel.result["body"]) + + # Attempt to access the media + request, channel = self.make_request( + "GET", + server_name_and_media_id, + shorthand=False, + access_token=admin_user_tok, + ) + request.render(self.download_resource) + self.pump(1.0) + + # Should be quarantined + self.assertEqual( + 404, + int(channel.code), + msg=( + "Expected to receive a 404 on accessing quarantined media: %s" + % server_name_and_media_id + ), + ) + + def test_quarantine_all_media_in_room(self): + self.register_user("room_admin", "pass", admin=True) + admin_user_tok = self.login("room_admin", "pass") + + non_admin_user = self.register_user("room_nonadmin", "pass", admin=False) + non_admin_user_tok = self.login("room_nonadmin", "pass") + + room_id = self.helper.create_room_as(non_admin_user, tok=admin_user_tok) + self.helper.join(room_id, non_admin_user, tok=non_admin_user_tok) + + # Upload some media + response_1 = self.helper.upload_media( + self.upload_resource, self.image_data, tok=non_admin_user_tok + ) + response_2 = self.helper.upload_media( + self.upload_resource, self.image_data, tok=non_admin_user_tok + ) + + # Extract mxcs + mxc_1 = response_1["content_uri"] + mxc_2 = response_2["content_uri"] + + # Send it into the room + self.helper.send_event( + room_id, + "m.room.message", + content={"body": "image-1", "msgtype": "m.image", "url": mxc_1}, + txn_id="111", + tok=non_admin_user_tok, + ) + self.helper.send_event( + room_id, + "m.room.message", + content={"body": "image-2", "msgtype": "m.image", "url": mxc_2}, + txn_id="222", + tok=non_admin_user_tok, + ) + + # Quarantine all media in the room + url = "/_synapse/admin/v1/room/%s/media/quarantine" % urllib.parse.quote( + room_id + ) + request, channel = self.make_request("POST", url, access_token=admin_user_tok,) + self.render(request) + self.pump(1.0) + self.assertEqual(200, int(channel.code), msg=channel.result["body"]) + self.assertEqual( + json.loads(channel.result["body"].decode("utf-8")), + {"num_quarantined": 2}, + "Expected 2 quarantined items", + ) + + # Convert mxc URLs to server/media_id strings + server_and_media_id_1 = mxc_1[6:] + server_and_media_id_2 = mxc_2[6:] + + # Test that we cannot download any of the media anymore + request, channel = self.make_request( + "GET", + server_and_media_id_1, + shorthand=False, + access_token=non_admin_user_tok, + ) + request.render(self.download_resource) + self.pump(1.0) + + # Should be quarantined + self.assertEqual( + 404, + int(channel.code), + msg=( + "Expected to receive a 404 on accessing quarantined media: %s" + % server_and_media_id_1 + ), + ) + + request, channel = self.make_request( + "GET", + server_and_media_id_2, + shorthand=False, + access_token=non_admin_user_tok, + ) + request.render(self.download_resource) + self.pump(1.0) + + # Should be quarantined + self.assertEqual( + 404, + int(channel.code), + msg=( + "Expected to receive a 404 on accessing quarantined media: %s" + % server_and_media_id_2 + ), + ) + + def test_quarantine_all_media_by_user(self): + self.register_user("user_admin", "pass", admin=True) + admin_user_tok = self.login("user_admin", "pass") + + non_admin_user = self.register_user("user_nonadmin", "pass", admin=False) + non_admin_user_tok = self.login("user_nonadmin", "pass") + + # Upload some media + response_1 = self.helper.upload_media( + self.upload_resource, self.image_data, tok=non_admin_user_tok + ) + response_2 = self.helper.upload_media( + self.upload_resource, self.image_data, tok=non_admin_user_tok + ) + + # Extract media IDs + server_and_media_id_1 = response_1["content_uri"][6:] + server_and_media_id_2 = response_2["content_uri"][6:] + + # Quarantine all media by this user + url = "/_synapse/admin/v1/user/%s/media/quarantine" % urllib.parse.quote( + non_admin_user + ) + request, channel = self.make_request( + "POST", url.encode("ascii"), access_token=admin_user_tok, + ) + self.render(request) + self.pump(1.0) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual( + json.loads(channel.result["body"].decode("utf-8")), + {"num_quarantined": 2}, + "Expected 2 quarantined items", + ) + + # Attempt to access each piece of media + request, channel = self.make_request( + "GET", + server_and_media_id_1, + shorthand=False, + access_token=non_admin_user_tok, + ) + request.render(self.download_resource) + self.pump(1.0) + + # Should be quarantined + self.assertEqual( + 404, + int(channel.code), + msg=( + "Expected to receive a 404 on accessing quarantined media: %s" + % server_and_media_id_1, + ), + ) + + # Attempt to access each piece of media + request, channel = self.make_request( + "GET", + server_and_media_id_2, + shorthand=False, + access_token=non_admin_user_tok, + ) + request.render(self.download_resource) + self.pump(1.0) + + # Should be quarantined + self.assertEqual( + 404, + int(channel.code), + msg=( + "Expected to receive a 404 on accessing quarantined media: %s" + % server_and_media_id_2 + ), + ) diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index e7417b3d14..873d5ef99c 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -21,6 +21,8 @@ import time import attr +from twisted.web.resource import Resource + from synapse.api.constants import Membership from tests.server import make_request, render @@ -160,3 +162,38 @@ class RestHelper(object): ) return channel.json_body + + def upload_media( + self, + resource: Resource, + image_data: bytes, + tok: str, + filename: str = "test.png", + expect_code: int = 200, + ) -> dict: + """Upload a piece of test media to the media repo + Args: + resource: The resource that will handle the upload request + image_data: The image data to upload + tok: The user token to use during the upload + filename: The filename of the media to be uploaded + expect_code: The return code to expect from attempting to upload the media + """ + image_length = len(image_data) + path = "/_matrix/media/r0/upload?filename=%s" % (filename,) + request, channel = make_request( + self.hs.get_reactor(), "POST", path, content=image_data, access_token=tok + ) + request.requestHeaders.addRawHeader( + b"Content-Length", str(image_length).encode("UTF-8") + ) + request.render(resource) + self.hs.get_reactor().pump([100]) + + assert channel.code == expect_code, "Expected: %d, got: %d, resp: %r" % ( + expect_code, + int(channel.result["code"]), + channel.result["body"], + ) + + return channel.json_body -- cgit 1.5.1