diff options
69 files changed, 3511 insertions, 1937 deletions
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 23b8d7f909..c239130c57 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -23,7 +23,12 @@ jobs: mdbook-version: '0.4.9' - name: Build the documentation - run: mdbook build + # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. + # However, we're using docs/README.md for other purposes and need to pick a new page + # as the default. Let's opt for the welcome page instead. + run: | + mdbook build + cp book/welcome_and_overview.html book/index.html # Deploy to the latest documentation directories - name: Deploy latest documentation diff --git a/CHANGES.md b/CHANGES.md index 7b6e052aca..0c64d5bda6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -775,7 +775,7 @@ Internal Changes Synapse 1.29.0 (2021-03-08) =========================== -Note that synapse now expects an `X-Forwarded-Proto` header when used with a reverse proxy. Please see [UPGRADE.rst](UPGRADE.rst#upgrading-to-v1290) for more details on this change. +Note that synapse now expects an `X-Forwarded-Proto` header when used with a reverse proxy. Please see the [upgrade notes](docs/upgrade.md#upgrading-to-v1290) for more details on this change. No significant changes. @@ -840,7 +840,7 @@ Synapse 1.28.0 (2021-02-25) Note that this release drops support for ARMv7 in the official Docker images, due to repeated problems building for ARMv7 (and the associated maintenance burden this entails). -This release also fixes the documentation included in v1.27.0 around the callback URI for SAML2 identity providers. If your server is configured to use single sign-on via a SAML2 IdP, you may need to make configuration changes. Please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes. +This release also fixes the documentation included in v1.27.0 around the callback URI for SAML2 identity providers. If your server is configured to use single sign-on via a SAML2 IdP, you may need to make configuration changes. Please review the [upgrade notes](docs/upgrade.md) for more details on these changes. Internal Changes @@ -939,9 +939,9 @@ Synapse 1.27.0 (2021-02-16) Note that this release includes a change in Synapse to use Redis as a cache ─ as well as a pub/sub mechanism ─ if Redis support is enabled for workers. No action is needed by server administrators, and we do not expect resource usage of the Redis instance to change dramatically. -This release also changes the callback URI for OpenID Connect (OIDC) and SAML2 identity providers. If your server is configured to use single sign-on via an OIDC/OAuth2 or SAML2 IdP, you may need to make configuration changes. Please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes. +This release also changes the callback URI for OpenID Connect (OIDC) and SAML2 identity providers. If your server is configured to use single sign-on via an OIDC/OAuth2 or SAML2 IdP, you may need to make configuration changes. Please review the [upgrade notes](docs/upgrade.md) for more details on these changes. -This release also changes escaping of variables in the HTML templates for SSO or email notifications. If you have customised these templates, please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes. +This release also changes escaping of variables in the HTML templates for SSO or email notifications. If you have customised these templates, please review the [upgrade notes](docs/upgrade.md) for more details on these changes. Bugfixes @@ -1045,7 +1045,7 @@ Synapse 1.26.0 (2021-01-27) =========================== This release brings a new schema version for Synapse and rolling back to a previous -version is not trivial. Please review [UPGRADE.rst](UPGRADE.rst) for more details +version is not trivial. Please review the [upgrade notes](docs/upgrade.md) for more details on these changes and for general upgrade guidance. No significant changes since 1.26.0rc2. @@ -1072,7 +1072,7 @@ Synapse 1.26.0rc1 (2021-01-20) ============================== This release brings a new schema version for Synapse and rolling back to a previous -version is not trivial. Please review [UPGRADE.rst](UPGRADE.rst) for more details +version is not trivial. Please review the [upgrade notes](docs/upgrade.md) for more details on these changes and for general upgrade guidance. Features @@ -1478,7 +1478,7 @@ Internal Changes Synapse 1.23.0 (2020-11-18) =========================== -This release changes the way structured logging is configured. See the [upgrade notes](UPGRADE.rst#upgrading-to-v1230) for details. +This release changes the way structured logging is configured. See the [upgrade notes](docs/upgrade.md#upgrading-to-v1230) for details. **Note**: We are aware of a trivially exploitable denial of service vulnerability in versions of Synapse prior to 1.20.0. Complete details will be disclosed on Monday, November 23rd. If you have not upgraded recently, please do so. @@ -2081,7 +2081,10 @@ No significant changes since 1.19.0rc1. Removal warning --------------- -As outlined in the [previous release](https://github.com/matrix-org/synapse/releases/tag/v1.18.0), we are no longer publishing Docker images with the `-py3` tag suffix. On top of that, we have also removed the `latest-py3` tag. Please see [the announcement in the upgrade notes for 1.18.0](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#upgrading-to-v1180). +As outlined in the [previous release](https://github.com/matrix-org/synapse/releases/tag/v1.18.0), +we are no longer publishing Docker images with the `-py3` tag suffix. On top of that, we have also removed the +`latest-py3` tag. Please see +[the announcement in the upgrade notes for 1.18.0](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1180). Synapse 1.19.0rc1 (2020-08-13) @@ -2112,7 +2115,7 @@ Bugfixes Updates to the Docker image --------------------------- -- We no longer publish Docker images with the `-py3` tag suffix, as [announced in the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#upgrading-to-v1180). ([\#8056](https://github.com/matrix-org/synapse/issues/8056)) +- We no longer publish Docker images with the `-py3` tag suffix, as [announced in the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1180). ([\#8056](https://github.com/matrix-org/synapse/issues/8056)) Improved Documentation @@ -2670,7 +2673,7 @@ configurations of Synapse: to be incomplete or empty if Synapse was upgraded directly from v1.2.1 or earlier, to versions between v1.4.0 and v1.12.x. -Please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes +Please review the [upgrade notes](docs/upgrade.md) for more details on these changes and for general upgrade guidance. @@ -2771,7 +2774,7 @@ Bugfixes - Fix bad error handling that would cause Synapse to crash if it's provided with a YAML configuration file that's either empty or doesn't parse into a key-value map. ([\#7341](https://github.com/matrix-org/synapse/issues/7341)) - Fix incorrect metrics reporting for `renew_attestations` background task. ([\#7344](https://github.com/matrix-org/synapse/issues/7344)) - Prevent non-federating rooms from appearing in responses to federated `POST /publicRoom` requests when a filter was included. ([\#7367](https://github.com/matrix-org/synapse/issues/7367)) -- Fix a bug which would cause the room durectory to be incorrectly populated if Synapse was upgraded directly from v1.2.1 or earlier to v1.4.0 or later. Note that this fix does not apply retrospectively; see the [upgrade notes](UPGRADE.rst#upgrading-to-v1130) for more information. ([\#7387](https://github.com/matrix-org/synapse/issues/7387)) +- Fix a bug which would cause the room durectory to be incorrectly populated if Synapse was upgraded directly from v1.2.1 or earlier to v1.4.0 or later. Note that this fix does not apply retrospectively; see the [upgrade notes](docs/upgrade.md#upgrading-to-v1130) for more information. ([\#7387](https://github.com/matrix-org/synapse/issues/7387)) - Fix bug in `EventContext.deserialize`. ([\#7393](https://github.com/matrix-org/synapse/issues/7393)) @@ -2921,7 +2924,7 @@ Synapse 1.12.0 includes a database update which is run as part of the upgrade, and which may take some time (several hours in the case of a large server). Synapse will not respond to HTTP requests while this update is taking place. For imformation on seeing if you are affected, and workaround if you -are, see the [upgrade notes](UPGRADE.rst#upgrading-to-v1120). +are, see the [upgrade notes](docs/upgrade.md#upgrading-to-v1120). Security advisory ----------------- @@ -3474,7 +3477,7 @@ Bugfixes Synapse 1.7.0 (2019-12-13) ========================== -This release changes the default settings so that only local authenticated users can query the server's room directory. See the [upgrade notes](UPGRADE.rst#upgrading-to-v170) for details. +This release changes the default settings so that only local authenticated users can query the server's room directory. See the [upgrade notes](docs/upgrade.md#upgrading-to-v170) for details. Support for SQLite versions before 3.11 is now deprecated. A future release will refuse to start if used with an SQLite version before 3.11. @@ -3838,7 +3841,7 @@ Synapse 1.4.0rc1 (2019-09-26) ============================= Note that this release includes significant changes around 3pid -verification. Administrators are reminded to review the [upgrade notes](UPGRADE.rst#upgrading-to-v140). +verification. Administrators are reminded to review the [upgrade notes](docs/upgrade.md#upgrading-to-v140). Features -------- @@ -4214,7 +4217,7 @@ Synapse 1.1.0 (2019-07-04) ========================== As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4. -See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details. +See the [upgrade notes](docs/upgrade.md#upgrading-to-v110) for more details. This release also deprecates the use of environment variables to configure the docker image. See the [docker README](https://github.com/matrix-org/synapse/blob/release-v1.1.0/docker/README.md#legacy-dynamic-configuration-file-support) @@ -4244,7 +4247,7 @@ Synapse 1.1.0rc1 (2019-07-02) ============================= As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4. -See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details. +See the [upgrade notes](docs/upgrade.md#upgrading-to-v110) for more details. Features -------- @@ -5016,7 +5019,7 @@ run on Python versions 3.5 or 3.6 (as well as 2.7). Support for Python 3.7 remains experimental. We recommend upgrading to Python 3, but make sure to read the [upgrade -notes](UPGRADE.rst#upgrading-to-v0340) when doing so. +notes](docs/upgrade.md#upgrading-to-v0340) when doing so. Features -------- diff --git a/README.rst b/README.rst index 2ecc93c8a7..6d3cf6c1a5 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ The overall architecture is:: ``#matrix:matrix.org`` is the official support room for Matrix, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html or -via IRC bridge at irc://irc.freenode.net/matrix. +via IRC bridge at irc://irc.libera.chat/matrix. Synapse is currently in rapid development, but as of version 0.5 we believe it is sufficiently stable to be run as an internet-facing service for real usage! @@ -186,11 +186,11 @@ impact to other applications will be minimal. Upgrading an existing Synapse ============================= -The instructions for upgrading synapse are in `UPGRADE.rst`_. +The instructions for upgrading synapse are in `the upgrade notes`_. Please check these instructions as upgrading may require extra steps for some versions of synapse. -.. _UPGRADE.rst: UPGRADE.rst +.. _the upgrade notes: https://matrix-org.github.io/synapse/develop/upgrade.html .. _reverse-proxy: diff --git a/UPGRADE.rst b/UPGRADE.rst index ee8b4fa60b..82548ac850 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,1341 +1,7 @@ Upgrading Synapse ================= -Before upgrading check if any special steps are required to upgrade from the -version you currently have installed to the current version of Synapse. The extra -instructions that may be required are listed later in this document. +This document has moved to the `Synapse documentation website <https://matrix-org.github.io/synapse/develop/upgrading>`_. +Please update your links. -* Check that your versions of Python and PostgreSQL are still supported. - - Synapse follows upstream lifecycles for `Python`_ and `PostgreSQL`_, and - removes support for versions which are no longer maintained. - - The website https://endoflife.date also offers convenient summaries. - - .. _Python: https://devguide.python.org/devcycle/#end-of-life-branches - .. _PostgreSQL: https://www.postgresql.org/support/versioning/ - -* If Synapse was installed using `prebuilt packages - <INSTALL.md#prebuilt-packages>`_, you will need to follow the normal process - for upgrading those packages. - -* If Synapse was installed from source, then: - - 1. Activate the virtualenv before upgrading. For example, if Synapse is - installed in a virtualenv in ``~/synapse/env`` then run: - - .. code:: bash - - source ~/synapse/env/bin/activate - - 2. If Synapse was installed using pip then upgrade to the latest version by - running: - - .. code:: bash - - pip install --upgrade matrix-synapse - - If Synapse was installed using git then upgrade to the latest version by - running: - - .. code:: bash - - git pull - pip install --upgrade . - - 3. Restart Synapse: - - .. code:: bash - - ./synctl restart - -To check whether your update was successful, you can check the running server -version with: - -.. code:: bash - - # you may need to replace 'localhost:8008' if synapse is not configured - # to listen on port 8008. - - curl http://localhost:8008/_synapse/admin/v1/server_version - -Rolling back to older versions ------------------------------- - -Rolling back to previous releases can be difficult, due to database schema -changes between releases. Where we have been able to test the rollback process, -this will be noted below. - -In general, you will need to undo any changes made during the upgrade process, -for example: - -* pip: - - .. code:: bash - - source env/bin/activate - # replace `1.3.0` accordingly: - pip install matrix-synapse==1.3.0 - -* Debian: - - .. code:: bash - - # replace `1.3.0` and `stretch` accordingly: - wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb - dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb - -Upgrading to v1.37.0 -==================== - -Deprecation of the current spam checker interface -------------------------------------------------- - -The current spam checker interface is deprecated in favour of a new generic modules system. -Authors of spam checker modules can refer to `this documentation <https://matrix-org.github.io/synapse/develop/modules.html#porting-an-existing-module-that-uses-the-old-interface>`_ -to update their modules. Synapse administrators can refer to `this documentation <https://matrix-org.github.io/synapse/develop/modules.html#using-modules>`_ -to update their configuration once the modules they are using have been updated. - -We plan to remove support for the current spam checker interface in August 2021. - -More module interfaces will be ported over to this new generic system in future versions -of Synapse. - - -Upgrading to v1.34.0 -==================== - -``room_invite_state_types`` configuration setting ------------------------------------------------ - -The ``room_invite_state_types`` configuration setting has been deprecated and -replaced with ``room_prejoin_state``. See the `sample configuration file <https://github.com/matrix-org/synapse/blob/v1.34.0/docs/sample_config.yaml#L1515>`_. - -If you have set ``room_invite_state_types`` to the default value you should simply -remove it from your configuration file. The default value used to be: - -.. code:: yaml - - room_invite_state_types: - - "m.room.join_rules" - - "m.room.canonical_alias" - - "m.room.avatar" - - "m.room.encryption" - - "m.room.name" - -If you have customised this value, you should remove ``room_invite_state_types`` and -configure ``room_prejoin_state`` instead. - - - -Upgrading to v1.33.0 -==================== - -Account Validity HTML templates can now display a user's expiration date ------------------------------------------------------------------------- - -This may affect you if you have enabled the account validity feature, and have made use of a -custom HTML template specified by the ``account_validity.template_dir`` or ``account_validity.account_renewed_html_path`` -Synapse config options. - -The template can now accept an ``expiration_ts`` variable, which represents the unix timestamp in milliseconds for the -future date of which their account has been renewed until. See the -`default template <https://github.com/matrix-org/synapse/blob/release-v1.33.0/synapse/res/templates/account_renewed.html>`_ -for an example of usage. - -ALso note that a new HTML template, ``account_previously_renewed.html``, has been added. This is is shown to users -when they attempt to renew their account with a valid renewal token that has already been used before. The default -template contents can been found -`here <https://github.com/matrix-org/synapse/blob/release-v1.33.0/synapse/res/templates/account_previously_renewed.html>`_, -and can also accept an ``expiration_ts`` variable. This template replaces the error message users would previously see -upon attempting to use a valid renewal token more than once. - - -Upgrading to v1.32.0 -==================== - -Regression causing connected Prometheus instances to become overwhelmed ------------------------------------------------------------------------ - -This release introduces `a regression <https://github.com/matrix-org/synapse/issues/9853>`_ -that can overwhelm connected Prometheus instances. This issue is not present in -Synapse v1.32.0rc1. - -If you have been affected, please downgrade to 1.31.0. You then may need to -remove excess writeahead logs in order for Prometheus to recover. Instructions -for doing so are provided -`here <https://github.com/matrix-org/synapse/pull/9854#issuecomment-823472183>`_. - -Dropping support for old Python, Postgres and SQLite versions -------------------------------------------------------------- - -In line with our `deprecation policy <https://github.com/matrix-org/synapse/blob/release-v1.32.0/docs/deprecation_policy.md>`_, -we've dropped support for Python 3.5 and PostgreSQL 9.5, as they are no longer supported upstream. - -This release of Synapse requires Python 3.6+ and PostgresSQL 9.6+ or SQLite 3.22+. - -Removal of old List Accounts Admin API --------------------------------------- - -The deprecated v1 "list accounts" admin API (``GET /_synapse/admin/v1/users/<user_id>``) has been removed in this version. - -The `v2 list accounts API <https://github.com/matrix-org/synapse/blob/master/docs/admin_api/user_admin_api.rst#list-accounts>`_ -has been available since Synapse 1.7.0 (2019-12-13), and is accessible under ``GET /_synapse/admin/v2/users``. - -The deprecation of the old endpoint was announced with Synapse 1.28.0 (released on 2021-02-25). - -Application Services must use type ``m.login.application_service`` when registering users ------------------------------------------------------------------------------------------ - -In compliance with the -`Application Service spec <https://matrix.org/docs/spec/application_service/r0.1.2#server-admin-style-permissions>`_, -Application Services are now required to use the ``m.login.application_service`` type when registering users via the -``/_matrix/client/r0/register`` endpoint. This behaviour was deprecated in Synapse v1.30.0. - -Please ensure your Application Services are up to date. - -Upgrading to v1.29.0 -==================== - -Requirement for X-Forwarded-Proto header ----------------------------------------- - -When using Synapse with a reverse proxy (in particular, when using the -`x_forwarded` option on an HTTP listener), Synapse now expects to receive an -`X-Forwarded-Proto` header on incoming HTTP requests. If it is not set, Synapse -will log a warning on each received request. - -To avoid the warning, administrators using a reverse proxy should ensure that -the reverse proxy sets `X-Forwarded-Proto` header to `https` or `http` to -indicate the protocol used by the client. - -Synapse also requires the `Host` header to be preserved. - -See the `reverse proxy documentation <docs/reverse_proxy.md>`_, where the -example configurations have been updated to show how to set these headers. - -(Users of `Caddy <https://caddyserver.com/>`_ are unaffected, since we believe it -sets `X-Forwarded-Proto` by default.) - -Upgrading to v1.27.0 -==================== - -Changes to callback URI for OAuth2 / OpenID Connect and SAML2 -------------------------------------------------------------- - -This version changes the URI used for callbacks from OAuth2 and SAML2 identity providers: - -* If your server is configured for single sign-on via an OpenID Connect or OAuth2 identity - provider, you will need to add ``[synapse public baseurl]/_synapse/client/oidc/callback`` - to the list of permitted "redirect URIs" at the identity provider. - - See `docs/openid.md <docs/openid.md>`_ for more information on setting up OpenID - Connect. - -* If your server is configured for single sign-on via a SAML2 identity provider, you will - need to add ``[synapse public baseurl]/_synapse/client/saml2/authn_response`` as a permitted - "ACS location" (also known as "allowed callback URLs") at the identity provider. - - The "Issuer" in the "AuthnRequest" to the SAML2 identity provider is also updated to - ``[synapse public baseurl]/_synapse/client/saml2/metadata.xml``. If your SAML2 identity - provider uses this property to validate or otherwise identify Synapse, its configuration - will need to be updated to use the new URL. Alternatively you could create a new, separate - "EntityDescriptor" in your SAML2 identity provider with the new URLs and leave the URLs in - the existing "EntityDescriptor" as they were. - -Changes to HTML templates -------------------------- - -The HTML templates for SSO and email notifications now have `Jinja2's autoescape <https://jinja.palletsprojects.com/en/2.11.x/api/#autoescaping>`_ -enabled for files ending in ``.html``, ``.htm``, and ``.xml``. If you have customised -these templates and see issues when viewing them you might need to update them. -It is expected that most configurations will need no changes. - -If you have customised the templates *names* for these templates, it is recommended -to verify they end in ``.html`` to ensure autoescape is enabled. - -The above applies to the following templates: - -* ``add_threepid.html`` -* ``add_threepid_failure.html`` -* ``add_threepid_success.html`` -* ``notice_expiry.html`` -* ``notice_expiry.html`` -* ``notif_mail.html`` (which, by default, includes ``room.html`` and ``notif.html``) -* ``password_reset.html`` -* ``password_reset_confirmation.html`` -* ``password_reset_failure.html`` -* ``password_reset_success.html`` -* ``registration.html`` -* ``registration_failure.html`` -* ``registration_success.html`` -* ``sso_account_deactivated.html`` -* ``sso_auth_bad_user.html`` -* ``sso_auth_confirm.html`` -* ``sso_auth_success.html`` -* ``sso_error.html`` -* ``sso_login_idp_picker.html`` -* ``sso_redirect_confirm.html`` - -Upgrading to v1.26.0 -==================== - -Rolling back to v1.25.0 after a failed upgrade ----------------------------------------------- - -v1.26.0 includes a lot of large changes. If something problematic occurs, you -may want to roll-back to a previous version of Synapse. Because v1.26.0 also -includes a new database schema version, reverting that version is also required -alongside the generic rollback instructions mentioned above. In short, to roll -back to v1.25.0 you need to: - -1. Stop the server -2. Decrease the schema version in the database: - - .. code:: sql - - UPDATE schema_version SET version = 58; - -3. Delete the ignored users & chain cover data: - - .. code:: sql - - DROP TABLE IF EXISTS ignored_users; - UPDATE rooms SET has_auth_chain_index = false; - - For PostgreSQL run: - - .. code:: sql - - TRUNCATE event_auth_chain_links; - TRUNCATE event_auth_chains; - - For SQLite run: - - .. code:: sql - - DELETE FROM event_auth_chain_links; - DELETE FROM event_auth_chains; - -4. Mark the deltas as not run (so they will re-run on upgrade). - - .. code:: sql - - DELETE FROM applied_schema_deltas WHERE version = 59 AND file = "59/01ignored_user.py"; - DELETE FROM applied_schema_deltas WHERE version = 59 AND file = "59/06chain_cover_index.sql"; - -5. Downgrade Synapse by following the instructions for your installation method - in the "Rolling back to older versions" section above. - -Upgrading to v1.25.0 -==================== - -Last release supporting Python 3.5 ----------------------------------- - -This is the last release of Synapse which guarantees support with Python 3.5, -which passed its upstream End of Life date several months ago. - -We will attempt to maintain support through March 2021, but without guarantees. - -In the future, Synapse will follow upstream schedules for ending support of -older versions of Python and PostgreSQL. Please upgrade to at least Python 3.6 -and PostgreSQL 9.6 as soon as possible. - -Blacklisting IP ranges ----------------------- - -Synapse v1.25.0 includes new settings, ``ip_range_blacklist`` and -``ip_range_whitelist``, for controlling outgoing requests from Synapse for federation, -identity servers, push, and for checking key validity for third-party invite events. -The previous setting, ``federation_ip_range_blacklist``, is deprecated. The new -``ip_range_blacklist`` defaults to private IP ranges if it is not defined. - -If you have never customised ``federation_ip_range_blacklist`` it is recommended -that you remove that setting. - -If you have customised ``federation_ip_range_blacklist`` you should update the -setting name to ``ip_range_blacklist``. - -If you have a custom push server that is reached via private IP space you may -need to customise ``ip_range_blacklist`` or ``ip_range_whitelist``. - -Upgrading to v1.24.0 -==================== - -Custom OpenID Connect mapping provider breaking change ------------------------------------------------------- - -This release allows the OpenID Connect mapping provider to perform normalisation -of the localpart of the Matrix ID. This allows for the mapping provider to -specify different algorithms, instead of the [default way](https://matrix.org/docs/spec/appendices#mapping-from-other-character-sets). - -If your Synapse configuration uses a custom mapping provider -(`oidc_config.user_mapping_provider.module` is specified and not equal to -`synapse.handlers.oidc_handler.JinjaOidcMappingProvider`) then you *must* ensure -that `map_user_attributes` of the mapping provider performs some normalisation -of the `localpart` returned. To match previous behaviour you can use the -`map_username_to_mxid_localpart` function provided by Synapse. An example is -shown below: - -.. code-block:: python - - from synapse.types import map_username_to_mxid_localpart - - class MyMappingProvider: - def map_user_attributes(self, userinfo, token): - # ... your custom logic ... - sso_user_id = ... - localpart = map_username_to_mxid_localpart(sso_user_id) - - return {"localpart": localpart} - -Removal historical Synapse Admin API ------------------------------------- - -Historically, the Synapse Admin API has been accessible under: - -* ``/_matrix/client/api/v1/admin`` -* ``/_matrix/client/unstable/admin`` -* ``/_matrix/client/r0/admin`` -* ``/_synapse/admin/v1`` - -The endpoints with ``/_matrix/client/*`` prefixes have been removed as of v1.24.0. -The Admin API is now only accessible under: - -* ``/_synapse/admin/v1`` - -The only exception is the `/admin/whois` endpoint, which is -`also available via the client-server API <https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid>`_. - -The deprecation of the old endpoints was announced with Synapse 1.20.0 (released -on 2020-09-22) and makes it easier for homeserver admins to lock down external -access to the Admin API endpoints. - -Upgrading to v1.23.0 -==================== - -Structured logging configuration breaking changes -------------------------------------------------- - -This release deprecates use of the ``structured: true`` logging configuration for -structured logging. If your logging configuration contains ``structured: true`` -then it should be modified based on the `structured logging documentation -<https://github.com/matrix-org/synapse/blob/master/docs/structured_logging.md>`_. - -The ``structured`` and ``drains`` logging options are now deprecated and should -be replaced by standard logging configuration of ``handlers`` and ``formatters``. - -A future will release of Synapse will make using ``structured: true`` an error. - -Upgrading to v1.22.0 -==================== - -ThirdPartyEventRules breaking changes -------------------------------------- - -This release introduces a backwards-incompatible change to modules making use of -``ThirdPartyEventRules`` in Synapse. If you make use of a module defined under the -``third_party_event_rules`` config option, please make sure it is updated to handle -the below change: - -The ``http_client`` argument is no longer passed to modules as they are initialised. Instead, -modules are expected to make use of the ``http_client`` property on the ``ModuleApi`` class. -Modules are now passed a ``module_api`` argument during initialisation, which is an instance of -``ModuleApi``. ``ModuleApi`` instances have a ``http_client`` property which acts the same as -the ``http_client`` argument previously passed to ``ThirdPartyEventRules`` modules. - -Upgrading to v1.21.0 -==================== - -Forwarding ``/_synapse/client`` through your reverse proxy ----------------------------------------------------------- - -The `reverse proxy documentation -<https://github.com/matrix-org/synapse/blob/develop/docs/reverse_proxy.md>`_ has been updated -to include reverse proxy directives for ``/_synapse/client/*`` endpoints. As the user password -reset flow now uses endpoints under this prefix, **you must update your reverse proxy -configurations for user password reset to work**. - -Additionally, note that the `Synapse worker documentation -<https://github.com/matrix-org/synapse/blob/develop/docs/workers.md>`_ has been updated to - state that the ``/_synapse/client/password_reset/email/submit_token`` endpoint can be handled -by all workers. If you make use of Synapse's worker feature, please update your reverse proxy -configuration to reflect this change. - -New HTML templates ------------------- - -A new HTML template, -`password_reset_confirmation.html <https://github.com/matrix-org/synapse/blob/develop/synapse/res/templates/password_reset_confirmation.html>`_, -has been added to the ``synapse/res/templates`` directory. If you are using a -custom template directory, you may want to copy the template over and modify it. - -Note that as of v1.20.0, templates do not need to be included in custom template -directories for Synapse to start. The default templates will be used if a custom -template cannot be found. - -This page will appear to the user after clicking a password reset link that has -been emailed to them. - -To complete password reset, the page must include a way to make a `POST` -request to -``/_synapse/client/password_reset/{medium}/submit_token`` -with the query parameters from the original link, presented as a URL-encoded form. See the file -itself for more details. - -Updated Single Sign-on HTML Templates -------------------------------------- - -The ``saml_error.html`` template was removed from Synapse and replaced with the -``sso_error.html`` template. If your Synapse is configured to use SAML and a -custom ``sso_redirect_confirm_template_dir`` configuration then any customisations -of the ``saml_error.html`` template will need to be merged into the ``sso_error.html`` -template. These templates are similar, but the parameters are slightly different: - -* The ``msg`` parameter should be renamed to ``error_description``. -* There is no longer a ``code`` parameter for the response code. -* A string ``error`` parameter is available that includes a short hint of why a - user is seeing the error page. - -Upgrading to v1.18.0 -==================== - -Docker `-py3` suffix will be removed in future versions -------------------------------------------------------- - -From 10th August 2020, we will no longer publish Docker images with the `-py3` tag suffix. The images tagged with the `-py3` suffix have been identical to the non-suffixed tags since release 0.99.0, and the suffix is obsolete. - -On 10th August, we will remove the `latest-py3` tag. Existing per-release tags (such as `v1.18.0-py3`) will not be removed, but no new `-py3` tags will be added. - -Scripts relying on the `-py3` suffix will need to be updated. - -Redis replication is now recommended in lieu of TCP replication ---------------------------------------------------------------- - -When setting up worker processes, we now recommend the use of a Redis server for replication. **The old direct TCP connection method is deprecated and will be removed in a future release.** -See `docs/workers.md <docs/workers.md>`_ for more details. - -Upgrading to v1.14.0 -==================== - -This version includes a database update which is run as part of the upgrade, -and which may take a couple of minutes in the case of a large server. Synapse -will not respond to HTTP requests while this update is taking place. - -Upgrading to v1.13.0 -==================== - -Incorrect database migration in old synapse versions ----------------------------------------------------- - -A bug was introduced in Synapse 1.4.0 which could cause the room directory to -be incomplete or empty if Synapse was upgraded directly from v1.2.1 or -earlier, to versions between v1.4.0 and v1.12.x. - -This will *not* be a problem for Synapse installations which were: - * created at v1.4.0 or later, - * upgraded via v1.3.x, or - * upgraded straight from v1.2.1 or earlier to v1.13.0 or later. - -If completeness of the room directory is a concern, installations which are -affected can be repaired as follows: - -1. Run the following sql from a `psql` or `sqlite3` console: - - .. code:: sql - - INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES - ('populate_stats_process_rooms', '{}', 'current_state_events_membership'); - - INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES - ('populate_stats_process_users', '{}', 'populate_stats_process_rooms'); - -2. Restart synapse. - -New Single Sign-on HTML Templates ---------------------------------- - -New templates (``sso_auth_confirm.html``, ``sso_auth_success.html``, and -``sso_account_deactivated.html``) were added to Synapse. If your Synapse is -configured to use SSO and a custom ``sso_redirect_confirm_template_dir`` -configuration then these templates will need to be copied from -`synapse/res/templates <synapse/res/templates>`_ into that directory. - -Synapse SSO Plugins Method Deprecation --------------------------------------- - -Plugins using the ``complete_sso_login`` method of -``synapse.module_api.ModuleApi`` should update to using the async/await -version ``complete_sso_login_async`` which includes additional checks. The -non-async version is considered deprecated. - -Rolling back to v1.12.4 after a failed upgrade ----------------------------------------------- - -v1.13.0 includes a lot of large changes. If something problematic occurs, you -may want to roll-back to a previous version of Synapse. Because v1.13.0 also -includes a new database schema version, reverting that version is also required -alongside the generic rollback instructions mentioned above. In short, to roll -back to v1.12.4 you need to: - -1. Stop the server -2. Decrease the schema version in the database: - - .. code:: sql - - UPDATE schema_version SET version = 57; - -3. Downgrade Synapse by following the instructions for your installation method - in the "Rolling back to older versions" section above. - - -Upgrading to v1.12.0 -==================== - -This version includes a database update which is run as part of the upgrade, -and which may take some time (several hours in the case of a large -server). Synapse will not respond to HTTP requests while this update is taking -place. - -This is only likely to be a problem in the case of a server which is -participating in many rooms. - -0. As with all upgrades, it is recommended that you have a recent backup of - your database which can be used for recovery in the event of any problems. - -1. As an initial check to see if you will be affected, you can try running the - following query from the `psql` or `sqlite3` console. It is safe to run it - while Synapse is still running. - - .. code:: sql - - SELECT MAX(q.v) FROM ( - SELECT ( - SELECT ej.json AS v - FROM state_events se INNER JOIN event_json ej USING (event_id) - WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key='' - LIMIT 1 - ) FROM rooms WHERE rooms.room_version IS NULL - ) q; - - This query will take about the same amount of time as the upgrade process: ie, - if it takes 5 minutes, then it is likely that Synapse will be unresponsive for - 5 minutes during the upgrade. - - If you consider an outage of this duration to be acceptable, no further - action is necessary and you can simply start Synapse 1.12.0. - - If you would prefer to reduce the downtime, continue with the steps below. - -2. The easiest workaround for this issue is to manually - create a new index before upgrading. On PostgreSQL, his can be done as follows: - - .. code:: sql - - CREATE INDEX CONCURRENTLY tmp_upgrade_1_12_0_index - ON state_events(room_id) WHERE type = 'm.room.create'; - - The above query may take some time, but is also safe to run while Synapse is - running. - - We assume that no SQLite users have databases large enough to be - affected. If you *are* affected, you can run a similar query, omitting the - ``CONCURRENTLY`` keyword. Note however that this operation may in itself cause - Synapse to stop running for some time. Synapse admins are reminded that - `SQLite is not recommended for use outside a test - environment <https://github.com/matrix-org/synapse/blob/master/README.rst#using-postgresql>`_. - -3. Once the index has been created, the ``SELECT`` query in step 1 above should - complete quickly. It is therefore safe to upgrade to Synapse 1.12.0. - -4. Once Synapse 1.12.0 has successfully started and is responding to HTTP - requests, the temporary index can be removed: - - .. code:: sql - - DROP INDEX tmp_upgrade_1_12_0_index; - -Upgrading to v1.10.0 -==================== - -Synapse will now log a warning on start up if used with a PostgreSQL database -that has a non-recommended locale set. - -See `docs/postgres.md <docs/postgres.md>`_ for details. - - -Upgrading to v1.8.0 -=================== - -Specifying a ``log_file`` config option will now cause Synapse to refuse to -start, and should be replaced by with the ``log_config`` option. Support for -the ``log_file`` option was removed in v1.3.0 and has since had no effect. - - -Upgrading to v1.7.0 -=================== - -In an attempt to configure Synapse in a privacy preserving way, the default -behaviours of ``allow_public_rooms_without_auth`` and -``allow_public_rooms_over_federation`` have been inverted. This means that by -default, only authenticated users querying the Client/Server API will be able -to query the room directory, and relatedly that the server will not share -room directory information with other servers over federation. - -If your installation does not explicitly set these settings one way or the other -and you want either setting to be ``true`` then it will necessary to update -your homeserver configuration file accordingly. - -For more details on the surrounding context see our `explainer -<https://matrix.org/blog/2019/11/09/avoiding-unwelcome-visitors-on-private-matrix-servers>`_. - - -Upgrading to v1.5.0 -=================== - -This release includes a database migration which may take several minutes to -complete if there are a large number (more than a million or so) of entries in -the ``devices`` table. This is only likely to a be a problem on very large -installations. - - -Upgrading to v1.4.0 -=================== - -New custom templates --------------------- - -If you have configured a custom template directory with the -``email.template_dir`` option, be aware that there are new templates regarding -registration and threepid management (see below) that must be included. - -* ``registration.html`` and ``registration.txt`` -* ``registration_success.html`` and ``registration_failure.html`` -* ``add_threepid.html`` and ``add_threepid.txt`` -* ``add_threepid_failure.html`` and ``add_threepid_success.html`` - -Synapse will expect these files to exist inside the configured template -directory, and **will fail to start** if they are absent. -To view the default templates, see `synapse/res/templates -<https://github.com/matrix-org/synapse/tree/master/synapse/res/templates>`_. - -3pid verification changes -------------------------- - -**Note: As of this release, users will be unable to add phone numbers or email -addresses to their accounts, without changes to the Synapse configuration. This -includes adding an email address during registration.** - -It is possible for a user to associate an email address or phone number -with their account, for a number of reasons: - -* for use when logging in, as an alternative to the user id. -* in the case of email, as an alternative contact to help with account recovery. -* in the case of email, to receive notifications of missed messages. - -Before an email address or phone number can be added to a user's account, -or before such an address is used to carry out a password-reset, Synapse must -confirm the operation with the owner of the email address or phone number. -It does this by sending an email or text giving the user a link or token to confirm -receipt. This process is known as '3pid verification'. ('3pid', or 'threepid', -stands for third-party identifier, and we use it to refer to external -identifiers such as email addresses and phone numbers.) - -Previous versions of Synapse delegated the task of 3pid verification to an -identity server by default. In most cases this server is ``vector.im`` or -``matrix.org``. - -In Synapse 1.4.0, for security and privacy reasons, the homeserver will no -longer delegate this task to an identity server by default. Instead, -the server administrator will need to explicitly decide how they would like the -verification messages to be sent. - -In the medium term, the ``vector.im`` and ``matrix.org`` identity servers will -disable support for delegated 3pid verification entirely. However, in order to -ease the transition, they will retain the capability for a limited -period. Delegated email verification will be disabled on Monday 2nd December -2019 (giving roughly 2 months notice). Disabling delegated SMS verification -will follow some time after that once SMS verification support lands in -Synapse. - -Once delegated 3pid verification support has been disabled in the ``vector.im`` and -``matrix.org`` identity servers, all Synapse versions that depend on those -instances will be unable to verify email and phone numbers through them. There -are no imminent plans to remove delegated 3pid verification from Sydent -generally. (Sydent is the identity server project that backs the ``vector.im`` and -``matrix.org`` instances). - -Email -~~~~~ -Following upgrade, to continue verifying email (e.g. as part of the -registration process), admins can either:- - -* Configure Synapse to use an email server. -* Run or choose an identity server which allows delegated email verification - and delegate to it. - -Configure SMTP in Synapse -+++++++++++++++++++++++++ - -To configure an SMTP server for Synapse, modify the configuration section -headed ``email``, and be sure to have at least the ``smtp_host, smtp_port`` -and ``notif_from`` fields filled out. - -You may also need to set ``smtp_user``, ``smtp_pass``, and -``require_transport_security``. - -See the `sample configuration file <docs/sample_config.yaml>`_ for more details -on these settings. - -Delegate email to an identity server -++++++++++++++++++++++++++++++++++++ - -Some admins will wish to continue using email verification as part of the -registration process, but will not immediately have an appropriate SMTP server -at hand. - -To this end, we will continue to support email verification delegation via the -``vector.im`` and ``matrix.org`` identity servers for two months. Support for -delegated email verification will be disabled on Monday 2nd December. - -The ``account_threepid_delegates`` dictionary defines whether the homeserver -should delegate an external server (typically an `identity server -<https://matrix.org/docs/spec/identity_service/r0.2.1>`_) to handle sending -confirmation messages via email and SMS. - -So to delegate email verification, in ``homeserver.yaml``, set -``account_threepid_delegates.email`` to the base URL of an identity server. For -example: - -.. code:: yaml - - account_threepid_delegates: - email: https://example.com # Delegate email sending to example.com - -Note that ``account_threepid_delegates.email`` replaces the deprecated -``email.trust_identity_server_for_password_resets``: if -``email.trust_identity_server_for_password_resets`` is set to ``true``, and -``account_threepid_delegates.email`` is not set, then the first entry in -``trusted_third_party_id_servers`` will be used as the -``account_threepid_delegate`` for email. This is to ensure compatibility with -existing Synapse installs that set up external server handling for these tasks -before v1.4.0. If ``email.trust_identity_server_for_password_resets`` is -``true`` and no trusted identity server domains are configured, Synapse will -report an error and refuse to start. - -If ``email.trust_identity_server_for_password_resets`` is ``false`` or absent -and no ``email`` delegate is configured in ``account_threepid_delegates``, -then Synapse will send email verification messages itself, using the configured -SMTP server (see above). -that type. - -Phone numbers -~~~~~~~~~~~~~ - -Synapse does not support phone-number verification itself, so the only way to -maintain the ability for users to add phone numbers to their accounts will be -by continuing to delegate phone number verification to the ``matrix.org`` and -``vector.im`` identity servers (or another identity server that supports SMS -sending). - -The ``account_threepid_delegates`` dictionary defines whether the homeserver -should delegate an external server (typically an `identity server -<https://matrix.org/docs/spec/identity_service/r0.2.1>`_) to handle sending -confirmation messages via email and SMS. - -So to delegate phone number verification, in ``homeserver.yaml``, set -``account_threepid_delegates.msisdn`` to the base URL of an identity -server. For example: - -.. code:: yaml - - account_threepid_delegates: - msisdn: https://example.com # Delegate sms sending to example.com - -The ``matrix.org`` and ``vector.im`` identity servers will continue to support -delegated phone number verification via SMS until such time as it is possible -for admins to configure their servers to perform phone number verification -directly. More details will follow in a future release. - -Rolling back to v1.3.1 ----------------------- - -If you encounter problems with v1.4.0, it should be possible to roll back to -v1.3.1, subject to the following: - -* The 'room statistics' engine was heavily reworked in this release (see - `#5971 <https://github.com/matrix-org/synapse/pull/5971>`_), including - significant changes to the database schema, which are not easily - reverted. This will cause the room statistics engine to stop updating when - you downgrade. - - The room statistics are essentially unused in v1.3.1 (in future versions of - Synapse, they will be used to populate the room directory), so there should - be no loss of functionality. However, the statistics engine will write errors - to the logs, which can be avoided by setting the following in - `homeserver.yaml`: - - .. code:: yaml - - stats: - enabled: false - - Don't forget to re-enable it when you upgrade again, in preparation for its - use in the room directory! - -Upgrading to v1.2.0 -=================== - -Some counter metrics have been renamed, with the old names deprecated. See -`the metrics documentation <docs/metrics-howto.md#renaming-of-metrics--deprecation-of-old-names-in-12>`_ -for details. - -Upgrading to v1.1.0 -=================== - -Synapse v1.1.0 removes support for older Python and PostgreSQL versions, as -outlined in `our deprecation notice <https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x>`_. - -Minimum Python Version ----------------------- - -Synapse v1.1.0 has a minimum Python requirement of Python 3.5. Python 3.6 or -Python 3.7 are recommended as they have improved internal string handling, -significantly reducing memory usage. - -If you use current versions of the Matrix.org-distributed Debian packages or -Docker images, action is not required. - -If you install Synapse in a Python virtual environment, please see "Upgrading to -v0.34.0" for notes on setting up a new virtualenv under Python 3. - -Minimum PostgreSQL Version --------------------------- - -If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above. -Please see the -`PostgreSQL documentation <https://www.postgresql.org/docs/11/upgrading.html>`_ -for more details on upgrading your database. - -Upgrading to v1.0 -================= - -Validation of TLS certificates ------------------------------- - -Synapse v1.0 is the first release to enforce -validation of TLS certificates for the federation API. It is therefore -essential that your certificates are correctly configured. See the `FAQ -<docs/MSC1711_certificates_FAQ.md>`_ for more information. - -Note, v1.0 installations will also no longer be able to federate with servers -that have not correctly configured their certificates. - -In rare cases, it may be desirable to disable certificate checking: for -example, it might be essential to be able to federate with a given legacy -server in a closed federation. This can be done in one of two ways:- - -* Configure the global switch ``federation_verify_certificates`` to ``false``. -* Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``. - -See the `sample configuration file <docs/sample_config.yaml>`_ -for more details on these settings. - -Email ------ -When a user requests a password reset, Synapse will send an email to the -user to confirm the request. - -Previous versions of Synapse delegated the job of sending this email to an -identity server. If the identity server was somehow malicious or became -compromised, it would be theoretically possible to hijack an account through -this means. - -Therefore, by default, Synapse v1.0 will send the confirmation email itself. If -Synapse is not configured with an SMTP server, password reset via email will be -disabled. - -To configure an SMTP server for Synapse, modify the configuration section -headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port`` -and ``notif_from`` fields filled out. You may also need to set ``smtp_user``, -``smtp_pass``, and ``require_transport_security``. - -If you are absolutely certain that you wish to continue using an identity -server for password resets, set ``trust_identity_server_for_password_resets`` to ``true``. - -See the `sample configuration file <docs/sample_config.yaml>`_ -for more details on these settings. - -New email templates ---------------- -Some new templates have been added to the default template directory for the purpose of the -homeserver sending its own password reset emails. If you have configured a custom -``template_dir`` in your Synapse config, these files will need to be added. - -``password_reset.html`` and ``password_reset.txt`` are HTML and plain text templates -respectively that contain the contents of what will be emailed to the user upon attempting to -reset their password via email. ``password_reset_success.html`` and -``password_reset_failure.html`` are HTML files that the content of which (assuming no redirect -URL is set) will be shown to the user after they attempt to click the link in the email sent -to them. - -Upgrading to v0.99.0 -==================== - -Please be aware that, before Synapse v1.0 is released around March 2019, you -will need to replace any self-signed certificates with those verified by a -root CA. Information on how to do so can be found at `the ACME docs -<docs/ACME.md>`_. - -For more information on configuring TLS certificates see the `FAQ <docs/MSC1711_certificates_FAQ.md>`_. - -Upgrading to v0.34.0 -==================== - -1. This release is the first to fully support Python 3. Synapse will now run on - Python versions 3.5, or 3.6 (as well as 2.7). We recommend switching to - Python 3, as it has been shown to give performance improvements. - - For users who have installed Synapse into a virtualenv, we recommend doing - this by creating a new virtualenv. For example:: - - virtualenv -p python3 ~/synapse/env3 - source ~/synapse/env3/bin/activate - pip install matrix-synapse - - You can then start synapse as normal, having activated the new virtualenv:: - - cd ~/synapse - source env3/bin/activate - synctl start - - Users who have installed from distribution packages should see the relevant - package documentation. See below for notes on Debian packages. - - * When upgrading to Python 3, you **must** make sure that your log files are - configured as UTF-8, by adding ``encoding: utf8`` to the - ``RotatingFileHandler`` configuration (if you have one) in your - ``<server>.log.config`` file. For example, if your ``log.config`` file - contains:: - - handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: precise - filename: homeserver.log - maxBytes: 104857600 - backupCount: 10 - filters: [context] - console: - class: logging.StreamHandler - formatter: precise - filters: [context] - - Then you should update this to be:: - - handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: precise - filename: homeserver.log - maxBytes: 104857600 - backupCount: 10 - filters: [context] - encoding: utf8 - console: - class: logging.StreamHandler - formatter: precise - filters: [context] - - There is no need to revert this change if downgrading to Python 2. - - We are also making available Debian packages which will run Synapse on - Python 3. You can switch to these packages with ``apt-get install - matrix-synapse-py3``, however, please read `debian/NEWS - <https://github.com/matrix-org/synapse/blob/release-v0.34.0/debian/NEWS>`_ - before doing so. The existing ``matrix-synapse`` packages will continue to - use Python 2 for the time being. - -2. This release removes the ``riot.im`` from the default list of trusted - identity servers. - - If ``riot.im`` is in your homeserver's list of - ``trusted_third_party_id_servers``, you should remove it. It was added in - case a hypothetical future identity server was put there. If you don't - remove it, users may be unable to deactivate their accounts. - -3. This release no longer installs the (unmaintained) Matrix Console web client - as part of the default installation. It is possible to re-enable it by - installing it separately and setting the ``web_client_location`` config - option, but please consider switching to another client. - -Upgrading to v0.33.7 -==================== - -This release removes the example email notification templates from -``res/templates`` (they are now internal to the python package). This should -only affect you if you (a) deploy your Synapse instance from a git checkout or -a github snapshot URL, and (b) have email notifications enabled. - -If you have email notifications enabled, you should ensure that -``email.template_dir`` is either configured to point at a directory where you -have installed customised templates, or leave it unset to use the default -templates. - -Upgrading to v0.27.3 -==================== - -This release expands the anonymous usage stats sent if the opt-in -``report_stats`` configuration is set to ``true``. We now capture RSS memory -and cpu use at a very coarse level. This requires administrators to install -the optional ``psutil`` python module. - -We would appreciate it if you could assist by ensuring this module is available -and ``report_stats`` is enabled. This will let us see if performance changes to -synapse are having an impact to the general community. - -Upgrading to v0.15.0 -==================== - -If you want to use the new URL previewing API (/_matrix/media/r0/preview_url) -then you have to explicitly enable it in the config and update your dependencies -dependencies. See README.rst for details. - - -Upgrading to v0.11.0 -==================== - -This release includes the option to send anonymous usage stats to matrix.org, -and requires that administrators explictly opt in or out by setting the -``report_stats`` option to either ``true`` or ``false``. - -We would really appreciate it if you could help our project out by reporting -anonymized usage statistics from your homeserver. Only very basic aggregate -data (e.g. number of users) will be reported, but it helps us to track the -growth of the Matrix community, and helps us to make Matrix a success, as well -as to convince other networks that they should peer with us. - - -Upgrading to v0.9.0 -=================== - -Application services have had a breaking API change in this version. - -They can no longer register themselves with a home server using the AS HTTP API. This -decision was made because a compromised application service with free reign to register -any regex in effect grants full read/write access to the home server if a regex of ``.*`` -is used. An attack where a compromised AS re-registers itself with ``.*`` was deemed too -big of a security risk to ignore, and so the ability to register with the HS remotely has -been removed. - -It has been replaced by specifying a list of application service registrations in -``homeserver.yaml``:: - - app_service_config_files: ["registration-01.yaml", "registration-02.yaml"] - -Where ``registration-01.yaml`` looks like:: - - url: <String> # e.g. "https://my.application.service.com" - as_token: <String> - hs_token: <String> - sender_localpart: <String> # This is a new field which denotes the user_id localpart when using the AS token - namespaces: - users: - - exclusive: <Boolean> - regex: <String> # e.g. "@prefix_.*" - aliases: - - exclusive: <Boolean> - regex: <String> - rooms: - - exclusive: <Boolean> - regex: <String> - -Upgrading to v0.8.0 -=================== - -Servers which use captchas will need to add their public key to:: - - static/client/register/register_config.js - - window.matrixRegistrationConfig = { - recaptcha_public_key: "YOUR_PUBLIC_KEY" - }; - -This is required in order to support registration fallback (typically used on -mobile devices). - - -Upgrading to v0.7.0 -=================== - -New dependencies are: - -- pydenticon -- simplejson -- syutil -- matrix-angular-sdk - -To pull in these dependencies in a virtual env, run:: - - python synapse/python_dependencies.py | xargs -n 1 pip install - -Upgrading to v0.6.0 -=================== - -To pull in new dependencies, run:: - - python setup.py develop --user - -This update includes a change to the database schema. To upgrade you first need -to upgrade the database by running:: - - python scripts/upgrade_db_to_v0.6.0.py <db> <server_name> <signing_key> - -Where `<db>` is the location of the database, `<server_name>` is the -server name as specified in the synapse configuration, and `<signing_key>` is -the location of the signing key as specified in the synapse configuration. - -This may take some time to complete. Failures of signatures and content hashes -can safely be ignored. - - -Upgrading to v0.5.1 -=================== - -Depending on precisely when you installed v0.5.0 you may have ended up with -a stale release of the reference matrix webclient installed as a python module. -To uninstall it and ensure you are depending on the latest module, please run:: - - $ pip uninstall syweb - -Upgrading to v0.5.0 -=================== - -The webclient has been split out into a seperate repository/pacakage in this -release. Before you restart your homeserver you will need to pull in the -webclient package by running:: - - python setup.py develop --user - -This release completely changes the database schema and so requires upgrading -it before starting the new version of the homeserver. - -The script "database-prepare-for-0.5.0.sh" should be used to upgrade the -database. This will save all user information, such as logins and profiles, -but will otherwise purge the database. This includes messages, which -rooms the home server was a member of and room alias mappings. - -If you would like to keep your history, please take a copy of your database -file and ask for help in #matrix:matrix.org. The upgrade process is, -unfortunately, non trivial and requires human intervention to resolve any -resulting conflicts during the upgrade process. - -Before running the command the homeserver should be first completely -shutdown. To run it, simply specify the location of the database, e.g.: - - ./scripts/database-prepare-for-0.5.0.sh "homeserver.db" - -Once this has successfully completed it will be safe to restart the -homeserver. You may notice that the homeserver takes a few seconds longer to -restart than usual as it reinitializes the database. - -On startup of the new version, users can either rejoin remote rooms using room -aliases or by being reinvited. Alternatively, if any other homeserver sends a -message to a room that the homeserver was previously in the local HS will -automatically rejoin the room. - -Upgrading to v0.4.0 -=================== - -This release needs an updated syutil version. Run:: - - python setup.py develop - -You will also need to upgrade your configuration as the signing key format has -changed. Run:: - - python -m synapse.app.homeserver --config-path <CONFIG> --generate-config - - -Upgrading to v0.3.0 -=================== - -This registration API now closely matches the login API. This introduces a bit -more backwards and forwards between the HS and the client, but this improves -the overall flexibility of the API. You can now GET on /register to retrieve a list -of valid registration flows. Upon choosing one, they are submitted in the same -way as login, e.g:: - - { - type: m.login.password, - user: foo, - password: bar - } - -The default HS supports 2 flows, with and without Identity Server email -authentication. Enabling captcha on the HS will add in an extra step to all -flows: ``m.login.recaptcha`` which must be completed before you can transition -to the next stage. There is a new login type: ``m.login.email.identity`` which -contains the ``threepidCreds`` key which were previously sent in the original -register request. For more information on this, see the specification. - -Web Client ----------- - -The VoIP specification has changed between v0.2.0 and v0.3.0. Users should -refresh any browser tabs to get the latest web client code. Users on -v0.2.0 of the web client will not be able to call those on v0.3.0 and -vice versa. - - -Upgrading to v0.2.0 -=================== - -The home server now requires setting up of SSL config before it can run. To -automatically generate default config use:: - - $ python synapse/app/homeserver.py \ - --server-name machine.my.domain.name \ - --bind-port 8448 \ - --config-path homeserver.config \ - --generate-config - -This config can be edited if desired, for example to specify a different SSL -certificate to use. Once done you can run the home server using:: - - $ python synapse/app/homeserver.py --config-path homeserver.config - -See the README.rst for more information. - -Also note that some config options have been renamed, including: - -- "host" to "server-name" -- "database" to "database-path" -- "port" to "bind-port" and "unsecure-port" - - -Upgrading to v0.0.1 -=================== - -This release completely changes the database schema and so requires upgrading -it before starting the new version of the homeserver. - -The script "database-prepare-for-0.0.1.sh" should be used to upgrade the -database. This will save all user information, such as logins and profiles, -but will otherwise purge the database. This includes messages, which -rooms the home server was a member of and room alias mappings. - -Before running the command the homeserver should be first completely -shutdown. To run it, simply specify the location of the database, e.g.: - - ./scripts/database-prepare-for-0.0.1.sh "homeserver.db" - -Once this has successfully completed it will be safe to restart the -homeserver. You may notice that the homeserver takes a few seconds longer to -restart than usual as it reinitializes the database. - -On startup of the new version, users can either rejoin remote rooms using room -aliases or by being reinvited. Alternatively, if any other homeserver sends a -message to a room that the homeserver was previously in the local HS will -automatically rejoin the room. +The markdown source is available in `docs/upgrade.md <docs/upgrade.md>`_. diff --git a/changelog.d/10114.misc b/changelog.d/10114.misc new file mode 100644 index 0000000000..808548f7c7 --- /dev/null +++ b/changelog.d/10114.misc @@ -0,0 +1 @@ +Drop Origin and Accept from the value of the Access-Control-Allow-Headers response header. diff --git a/changelog.d/10166.doc b/changelog.d/10166.doc new file mode 100644 index 0000000000..8d1710c132 --- /dev/null +++ b/changelog.d/10166.doc @@ -0,0 +1 @@ +Move the upgrade notes to [docs/upgrade.md](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md) and convert them to markdown. diff --git a/changelog.d/10213.misc b/changelog.d/10213.misc new file mode 100644 index 0000000000..9adb0fbd02 --- /dev/null +++ b/changelog.d/10213.misc @@ -0,0 +1 @@ +Add type hints to the federation servlets. diff --git a/changelog.d/10214.feature b/changelog.d/10214.feature new file mode 100644 index 0000000000..a3818c9d25 --- /dev/null +++ b/changelog.d/10214.feature @@ -0,0 +1 @@ +Omit empty fields from the `/sync` response. Contributed by @deepbluev7. \ No newline at end of file diff --git a/changelog.d/10223.bugfix b/changelog.d/10223.bugfix new file mode 100644 index 0000000000..4e42f6b608 --- /dev/null +++ b/changelog.d/10223.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug which meant that invite rejections and knocks were not sent out over federation in a timely manner. diff --git a/changelog.d/10225.feature b/changelog.d/10225.feature new file mode 100644 index 0000000000..d16f66ffe9 --- /dev/null +++ b/changelog.d/10225.feature @@ -0,0 +1 @@ +Improve validation on federation `send_{join,leave,knock}` endpoints. diff --git a/changelog.d/10237.misc b/changelog.d/10237.misc new file mode 100644 index 0000000000..d76c119a41 --- /dev/null +++ b/changelog.d/10237.misc @@ -0,0 +1 @@ +Improve the reliability of auto-joining remote rooms. diff --git a/changelog.d/10239.misc b/changelog.d/10239.misc new file mode 100644 index 0000000000..d05f1c4411 --- /dev/null +++ b/changelog.d/10239.misc @@ -0,0 +1 @@ +Update the release script to use the semver terminology and determine the release branch based on the next version. diff --git a/changelog.d/10242.doc b/changelog.d/10242.doc new file mode 100644 index 0000000000..2241b28547 --- /dev/null +++ b/changelog.d/10242.doc @@ -0,0 +1 @@ +Choose Welcome & Overview as the default page for synapse documentation website. diff --git a/changelog.d/10243.feature b/changelog.d/10243.feature new file mode 100644 index 0000000000..d16f66ffe9 --- /dev/null +++ b/changelog.d/10243.feature @@ -0,0 +1 @@ +Improve validation on federation `send_{join,leave,knock}` endpoints. diff --git a/changelog.d/10253.misc b/changelog.d/10253.misc new file mode 100644 index 0000000000..44d9217245 --- /dev/null +++ b/changelog.d/10253.misc @@ -0,0 +1 @@ +Fix type hints for computing auth events. diff --git a/changelog.d/10256.misc b/changelog.d/10256.misc new file mode 100644 index 0000000000..adef12fcb9 --- /dev/null +++ b/changelog.d/10256.misc @@ -0,0 +1 @@ +Improve the performance of the spaces summary endpoint by only recursing into spaces (and not rooms in general). diff --git a/changelog.d/10258.doc b/changelog.d/10258.doc new file mode 100644 index 0000000000..1549786c0c --- /dev/null +++ b/changelog.d/10258.doc @@ -0,0 +1 @@ +Adjust the URL in the README.rst file to point to irc.libera.chat. diff --git a/changelog.d/10263.feature b/changelog.d/10263.feature new file mode 100644 index 0000000000..7b1d2fe60f --- /dev/null +++ b/changelog.d/10263.feature @@ -0,0 +1 @@ +Mark events received over federation which fail a spam check as "soft-failed". diff --git a/changelog.d/10264.bugfix b/changelog.d/10264.bugfix new file mode 100644 index 0000000000..7ebda7cdc2 --- /dev/null +++ b/changelog.d/10264.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where Synapse would return errors after 2<sup>31</sup> events were handled by the server. diff --git a/changelog.d/10267.bugfix b/changelog.d/10267.bugfix new file mode 100644 index 0000000000..7ebda7cdc2 --- /dev/null +++ b/changelog.d/10267.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where Synapse would return errors after 2<sup>31</sup> events were handled by the server. diff --git a/changelog.d/10279.bugfix b/changelog.d/10279.bugfix new file mode 100644 index 0000000000..ac8b64ead9 --- /dev/null +++ b/changelog.d/10279.bugfix @@ -0,0 +1 @@ +Fix the prometheus `synapse_federation_server_pdu_process_time` metric. Broke in v1.37.1. diff --git a/changelog.d/9450.feature b/changelog.d/9450.feature new file mode 100644 index 0000000000..455936a41d --- /dev/null +++ b/changelog.d/9450.feature @@ -0,0 +1 @@ +Implement refresh tokens as specified by [MSC2918](https://github.com/matrix-org/matrix-doc/pull/2918). diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 98969bdd2d..db4ef1a44e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -11,7 +11,7 @@ - [Delegation](delegate.md) # Upgrading - - [Upgrading between Synapse Versions](upgrading/README.md) + - [Upgrading between Synapse Versions](upgrade.md) - [Upgrading from pre-Synapse 1.0](MSC1711_certificates_FAQ.md) # Usage diff --git a/docs/upgrade.md b/docs/upgrade.md new file mode 100644 index 0000000000..a44960c2b8 --- /dev/null +++ b/docs/upgrade.md @@ -0,0 +1,1353 @@ +# Upgrading Synapse + +Before upgrading check if any special steps are required to upgrade from +the version you currently have installed to the current version of +Synapse. The extra instructions that may be required are listed later in +this document. + +- Check that your versions of Python and PostgreSQL are still + supported. + + Synapse follows upstream lifecycles for [Python](https://endoflife.date/python) and + [PostgreSQL](https://endoflife.date/postgresql), and removes support for versions + which are no longer maintained. + + The website <https://endoflife.date> also offers convenient + summaries. + +- If Synapse was installed using [prebuilt + packages](../setup/INSTALL.md#prebuilt-packages), you will need to follow the + normal process for upgrading those packages. + +- If Synapse was installed from source, then: + + 1. Activate the virtualenv before upgrading. For example, if + Synapse is installed in a virtualenv in `~/synapse/env` then + run: + + ```bash + source ~/synapse/env/bin/activate + ``` + + 2. If Synapse was installed using pip then upgrade to the latest + version by running: + + ```bash + pip install --upgrade matrix-synapse + ``` + + If Synapse was installed using git then upgrade to the latest + version by running: + + ```bash + git pull + pip install --upgrade . + ``` + + 3. Restart Synapse: + + ```bash + ./synctl restart + ``` + +To check whether your update was successful, you can check the running +server version with: + +```bash +# you may need to replace 'localhost:8008' if synapse is not configured +# to listen on port 8008. + +curl http://localhost:8008/_synapse/admin/v1/server_version +``` + +## Rolling back to older versions + +Rolling back to previous releases can be difficult, due to database +schema changes between releases. Where we have been able to test the +rollback process, this will be noted below. + +In general, you will need to undo any changes made during the upgrade +process, for example: + +- pip: + + ```bash + source env/bin/activate + # replace `1.3.0` accordingly: + pip install matrix-synapse==1.3.0 + ``` + +- Debian: + + ```bash + # replace `1.3.0` and `stretch` accordingly: + wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb + dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb + ``` + +# Upgrading to v1.37.0 + +## Deprecation of the current spam checker interface + +The current spam checker interface is deprecated in favour of a new generic modules system. +Authors of spam checker modules can refer to [this +documentation](https://matrix-org.github.io/synapse/develop/modules.html#porting-an-existing-module-that-uses-the-old-interface) +to update their modules. Synapse administrators can refer to [this +documentation](https://matrix-org.github.io/synapse/develop/modules.html#using-modules) +to update their configuration once the modules they are using have been updated. + +We plan to remove support for the current spam checker interface in August 2021. + +More module interfaces will be ported over to this new generic system in future versions +of Synapse. + + +# Upgrading to v1.34.0 + +## `room_invite_state_types` configuration setting + +The `room_invite_state_types` configuration setting has been deprecated +and replaced with `room_prejoin_state`. See the [sample configuration +file](https://github.com/matrix-org/synapse/blob/v1.34.0/docs/sample_config.yaml#L1515). + +If you have set `room_invite_state_types` to the default value you +should simply remove it from your configuration file. The default value +used to be: + +```yaml +room_invite_state_types: + - "m.room.join_rules" + - "m.room.canonical_alias" + - "m.room.avatar" + - "m.room.encryption" + - "m.room.name" +``` + +If you have customised this value, you should remove +`room_invite_state_types` and configure `room_prejoin_state` instead. + +# Upgrading to v1.33.0 + +## Account Validity HTML templates can now display a user's expiration date + +This may affect you if you have enabled the account validity feature, +and have made use of a custom HTML template specified by the +`account_validity.template_dir` or +`account_validity.account_renewed_html_path` Synapse config options. + +The template can now accept an `expiration_ts` variable, which +represents the unix timestamp in milliseconds for the future date of +which their account has been renewed until. See the [default +template](https://github.com/matrix-org/synapse/blob/release-v1.33.0/synapse/res/templates/account_renewed.html) +for an example of usage. + +ALso note that a new HTML template, `account_previously_renewed.html`, +has been added. This is is shown to users when they attempt to renew +their account with a valid renewal token that has already been used +before. The default template contents can been found +[here](https://github.com/matrix-org/synapse/blob/release-v1.33.0/synapse/res/templates/account_previously_renewed.html), +and can also accept an `expiration_ts` variable. This template replaces +the error message users would previously see upon attempting to use a +valid renewal token more than once. + +# Upgrading to v1.32.0 + +## Regression causing connected Prometheus instances to become overwhelmed + +This release introduces [a +regression](https://github.com/matrix-org/synapse/issues/9853) that can +overwhelm connected Prometheus instances. This issue is not present in +Synapse v1.32.0rc1. + +If you have been affected, please downgrade to 1.31.0. You then may need +to remove excess writeahead logs in order for Prometheus to recover. +Instructions for doing so are provided +[here](https://github.com/matrix-org/synapse/pull/9854#issuecomment-823472183). + +## Dropping support for old Python, Postgres and SQLite versions + +In line with our [deprecation +policy](https://github.com/matrix-org/synapse/blob/release-v1.32.0/docs/deprecation_policy.md), +we've dropped support for Python 3.5 and PostgreSQL 9.5, as they are no +longer supported upstream. + +This release of Synapse requires Python 3.6+ and PostgresSQL 9.6+ or +SQLite 3.22+. + +## Removal of old List Accounts Admin API + +The deprecated v1 "list accounts" admin API +(`GET /_synapse/admin/v1/users/<user_id>`) has been removed in this +version. + +The [v2 list accounts +API](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/user_admin_api.rst#list-accounts) +has been available since Synapse 1.7.0 (2019-12-13), and is accessible +under `GET /_synapse/admin/v2/users`. + +The deprecation of the old endpoint was announced with Synapse 1.28.0 +(released on 2021-02-25). + +## Application Services must use type `m.login.application_service` when registering users + +In compliance with the [Application Service +spec](https://matrix.org/docs/spec/application_service/r0.1.2#server-admin-style-permissions), +Application Services are now required to use the +`m.login.application_service` type when registering users via the +`/_matrix/client/r0/register` endpoint. This behaviour was deprecated in +Synapse v1.30.0. + +Please ensure your Application Services are up to date. + +# Upgrading to v1.29.0 + +## Requirement for X-Forwarded-Proto header + +When using Synapse with a reverse proxy (in particular, when using the +[x_forwarded]{.title-ref} option on an HTTP listener), Synapse now +expects to receive an [X-Forwarded-Proto]{.title-ref} header on incoming +HTTP requests. If it is not set, Synapse will log a warning on each +received request. + +To avoid the warning, administrators using a reverse proxy should ensure +that the reverse proxy sets [X-Forwarded-Proto]{.title-ref} header to +[https]{.title-ref} or [http]{.title-ref} to indicate the protocol used +by the client. + +Synapse also requires the [Host]{.title-ref} header to be preserved. + +See the [reverse proxy documentation](../reverse_proxy.md), where the +example configurations have been updated to show how to set these +headers. + +(Users of [Caddy](https://caddyserver.com/) are unaffected, since we +believe it sets [X-Forwarded-Proto]{.title-ref} by default.) + +# Upgrading to v1.27.0 + +## Changes to callback URI for OAuth2 / OpenID Connect and SAML2 + +This version changes the URI used for callbacks from OAuth2 and SAML2 +identity providers: + +- If your server is configured for single sign-on via an OpenID + Connect or OAuth2 identity provider, you will need to add + `[synapse public baseurl]/_synapse/client/oidc/callback` to the list + of permitted "redirect URIs" at the identity provider. + + See the [OpenID docs](../openid.md) for more information on setting + up OpenID Connect. + +- If your server is configured for single sign-on via a SAML2 identity + provider, you will need to add + `[synapse public baseurl]/_synapse/client/saml2/authn_response` as a + permitted "ACS location" (also known as "allowed callback URLs") + at the identity provider. + + The "Issuer" in the "AuthnRequest" to the SAML2 identity + provider is also updated to + `[synapse public baseurl]/_synapse/client/saml2/metadata.xml`. If + your SAML2 identity provider uses this property to validate or + otherwise identify Synapse, its configuration will need to be + updated to use the new URL. Alternatively you could create a new, + separate "EntityDescriptor" in your SAML2 identity provider with + the new URLs and leave the URLs in the existing "EntityDescriptor" + as they were. + +## Changes to HTML templates + +The HTML templates for SSO and email notifications now have [Jinja2's +autoescape](https://jinja.palletsprojects.com/en/2.11.x/api/#autoescaping) +enabled for files ending in `.html`, `.htm`, and `.xml`. If you have +customised these templates and see issues when viewing them you might +need to update them. It is expected that most configurations will need +no changes. + +If you have customised the templates *names* for these templates, it is +recommended to verify they end in `.html` to ensure autoescape is +enabled. + +The above applies to the following templates: + +- `add_threepid.html` +- `add_threepid_failure.html` +- `add_threepid_success.html` +- `notice_expiry.html` +- `notice_expiry.html` +- `notif_mail.html` (which, by default, includes `room.html` and + `notif.html`) +- `password_reset.html` +- `password_reset_confirmation.html` +- `password_reset_failure.html` +- `password_reset_success.html` +- `registration.html` +- `registration_failure.html` +- `registration_success.html` +- `sso_account_deactivated.html` +- `sso_auth_bad_user.html` +- `sso_auth_confirm.html` +- `sso_auth_success.html` +- `sso_error.html` +- `sso_login_idp_picker.html` +- `sso_redirect_confirm.html` + +# Upgrading to v1.26.0 + +## Rolling back to v1.25.0 after a failed upgrade + +v1.26.0 includes a lot of large changes. If something problematic +occurs, you may want to roll-back to a previous version of Synapse. +Because v1.26.0 also includes a new database schema version, reverting +that version is also required alongside the generic rollback +instructions mentioned above. In short, to roll back to v1.25.0 you need +to: + +1. Stop the server + +2. Decrease the schema version in the database: + + ```sql + UPDATE schema_version SET version = 58; + ``` + +3. Delete the ignored users & chain cover data: + + ```sql + DROP TABLE IF EXISTS ignored_users; + UPDATE rooms SET has_auth_chain_index = false; + ``` + + For PostgreSQL run: + + ```sql + TRUNCATE event_auth_chain_links; + TRUNCATE event_auth_chains; + ``` + + For SQLite run: + + ```sql + DELETE FROM event_auth_chain_links; + DELETE FROM event_auth_chains; + ``` + +4. Mark the deltas as not run (so they will re-run on upgrade). + + ```sql + DELETE FROM applied_schema_deltas WHERE version = 59 AND file = "59/01ignored_user.py"; + DELETE FROM applied_schema_deltas WHERE version = 59 AND file = "59/06chain_cover_index.sql"; + ``` + +5. Downgrade Synapse by following the instructions for your + installation method in the "Rolling back to older versions" + section above. + +# Upgrading to v1.25.0 + +## Last release supporting Python 3.5 + +This is the last release of Synapse which guarantees support with Python +3.5, which passed its upstream End of Life date several months ago. + +We will attempt to maintain support through March 2021, but without +guarantees. + +In the future, Synapse will follow upstream schedules for ending support +of older versions of Python and PostgreSQL. Please upgrade to at least +Python 3.6 and PostgreSQL 9.6 as soon as possible. + +## Blacklisting IP ranges + +Synapse v1.25.0 includes new settings, `ip_range_blacklist` and +`ip_range_whitelist`, for controlling outgoing requests from Synapse for +federation, identity servers, push, and for checking key validity for +third-party invite events. The previous setting, +`federation_ip_range_blacklist`, is deprecated. The new +`ip_range_blacklist` defaults to private IP ranges if it is not defined. + +If you have never customised `federation_ip_range_blacklist` it is +recommended that you remove that setting. + +If you have customised `federation_ip_range_blacklist` you should update +the setting name to `ip_range_blacklist`. + +If you have a custom push server that is reached via private IP space +you may need to customise `ip_range_blacklist` or `ip_range_whitelist`. + +# Upgrading to v1.24.0 + +## Custom OpenID Connect mapping provider breaking change + +This release allows the OpenID Connect mapping provider to perform +normalisation of the localpart of the Matrix ID. This allows for the +mapping provider to specify different algorithms, instead of the +[default +way](<https://matrix.org/docs/spec/appendices#mapping-from-other-character-sets>). + +If your Synapse configuration uses a custom mapping provider +([oidc_config.user_mapping_provider.module]{.title-ref} is specified and +not equal to +[synapse.handlers.oidc_handler.JinjaOidcMappingProvider]{.title-ref}) +then you *must* ensure that [map_user_attributes]{.title-ref} of the +mapping provider performs some normalisation of the +[localpart]{.title-ref} returned. To match previous behaviour you can +use the [map_username_to_mxid_localpart]{.title-ref} function provided +by Synapse. An example is shown below: + +```python +from synapse.types import map_username_to_mxid_localpart + +class MyMappingProvider: + def map_user_attributes(self, userinfo, token): + # ... your custom logic ... + sso_user_id = ... + localpart = map_username_to_mxid_localpart(sso_user_id) + + return {"localpart": localpart} +``` + +## Removal historical Synapse Admin API + +Historically, the Synapse Admin API has been accessible under: + +- `/_matrix/client/api/v1/admin` +- `/_matrix/client/unstable/admin` +- `/_matrix/client/r0/admin` +- `/_synapse/admin/v1` + +The endpoints with `/_matrix/client/*` prefixes have been removed as of +v1.24.0. The Admin API is now only accessible under: + +- `/_synapse/admin/v1` + +The only exception is the [/admin/whois]{.title-ref} endpoint, which is +[also available via the client-server +API](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid). + +The deprecation of the old endpoints was announced with Synapse 1.20.0 +(released on 2020-09-22) and makes it easier for homeserver admins to +lock down external access to the Admin API endpoints. + +# Upgrading to v1.23.0 + +## Structured logging configuration breaking changes + +This release deprecates use of the `structured: true` logging +configuration for structured logging. If your logging configuration +contains `structured: true` then it should be modified based on the +[structured logging +documentation](../structured_logging.md). + +The `structured` and `drains` logging options are now deprecated and +should be replaced by standard logging configuration of `handlers` and +`formatters`. + +A future will release of Synapse will make using `structured: true` an +error. + +# Upgrading to v1.22.0 + +## ThirdPartyEventRules breaking changes + +This release introduces a backwards-incompatible change to modules +making use of `ThirdPartyEventRules` in Synapse. If you make use of a +module defined under the `third_party_event_rules` config option, please +make sure it is updated to handle the below change: + +The `http_client` argument is no longer passed to modules as they are +initialised. Instead, modules are expected to make use of the +`http_client` property on the `ModuleApi` class. Modules are now passed +a `module_api` argument during initialisation, which is an instance of +`ModuleApi`. `ModuleApi` instances have a `http_client` property which +acts the same as the `http_client` argument previously passed to +`ThirdPartyEventRules` modules. + +# Upgrading to v1.21.0 + +## Forwarding `/_synapse/client` through your reverse proxy + +The [reverse proxy +documentation](https://github.com/matrix-org/synapse/blob/develop/docs/reverse_proxy.md) +has been updated to include reverse proxy directives for +`/_synapse/client/*` endpoints. As the user password reset flow now uses +endpoints under this prefix, **you must update your reverse proxy +configurations for user password reset to work**. + +Additionally, note that the [Synapse worker documentation](https://github.com/matrix-org/synapse/blob/develop/docs/workers.md) has been updated to + +: state that the `/_synapse/client/password_reset/email/submit_token` + endpoint can be handled + +by all workers. If you make use of Synapse's worker feature, please +update your reverse proxy configuration to reflect this change. + +## New HTML templates + +A new HTML template, +[password_reset_confirmation.html](https://github.com/matrix-org/synapse/blob/develop/synapse/res/templates/password_reset_confirmation.html), +has been added to the `synapse/res/templates` directory. If you are +using a custom template directory, you may want to copy the template +over and modify it. + +Note that as of v1.20.0, templates do not need to be included in custom +template directories for Synapse to start. The default templates will be +used if a custom template cannot be found. + +This page will appear to the user after clicking a password reset link +that has been emailed to them. + +To complete password reset, the page must include a way to make a +[POST]{.title-ref} request to +`/_synapse/client/password_reset/{medium}/submit_token` with the query +parameters from the original link, presented as a URL-encoded form. See +the file itself for more details. + +## Updated Single Sign-on HTML Templates + +The `saml_error.html` template was removed from Synapse and replaced +with the `sso_error.html` template. If your Synapse is configured to use +SAML and a custom `sso_redirect_confirm_template_dir` configuration then +any customisations of the `saml_error.html` template will need to be +merged into the `sso_error.html` template. These templates are similar, +but the parameters are slightly different: + +- The `msg` parameter should be renamed to `error_description`. +- There is no longer a `code` parameter for the response code. +- A string `error` parameter is available that includes a short hint + of why a user is seeing the error page. + +# Upgrading to v1.18.0 + +## Docker [-py3]{.title-ref} suffix will be removed in future versions + +From 10th August 2020, we will no longer publish Docker images with the +[-py3]{.title-ref} tag suffix. The images tagged with the +[-py3]{.title-ref} suffix have been identical to the non-suffixed tags +since release 0.99.0, and the suffix is obsolete. + +On 10th August, we will remove the [latest-py3]{.title-ref} tag. +Existing per-release tags (such as [v1.18.0-py3]{.title-ref}) will not +be removed, but no new [-py3]{.title-ref} tags will be added. + +Scripts relying on the [-py3]{.title-ref} suffix will need to be +updated. + +## Redis replication is now recommended in lieu of TCP replication + +When setting up worker processes, we now recommend the use of a Redis +server for replication. **The old direct TCP connection method is +deprecated and will be removed in a future release.** See +[workers](../workers.md) for more details. + +# Upgrading to v1.14.0 + +This version includes a database update which is run as part of the +upgrade, and which may take a couple of minutes in the case of a large +server. Synapse will not respond to HTTP requests while this update is +taking place. + +# Upgrading to v1.13.0 + +## Incorrect database migration in old synapse versions + +A bug was introduced in Synapse 1.4.0 which could cause the room +directory to be incomplete or empty if Synapse was upgraded directly +from v1.2.1 or earlier, to versions between v1.4.0 and v1.12.x. + +This will *not* be a problem for Synapse installations which were: + +: - created at v1.4.0 or later, + - upgraded via v1.3.x, or + - upgraded straight from v1.2.1 or earlier to v1.13.0 or later. + +If completeness of the room directory is a concern, installations which +are affected can be repaired as follows: + +1. Run the following sql from a [psql]{.title-ref} or + [sqlite3]{.title-ref} console: + + ```sql + INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES + ('populate_stats_process_rooms', '{}', 'current_state_events_membership'); + + INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES + ('populate_stats_process_users', '{}', 'populate_stats_process_rooms'); + ``` + +2. Restart synapse. + +## New Single Sign-on HTML Templates + +New templates (`sso_auth_confirm.html`, `sso_auth_success.html`, and +`sso_account_deactivated.html`) were added to Synapse. If your Synapse +is configured to use SSO and a custom +`sso_redirect_confirm_template_dir` configuration then these templates +will need to be copied from +[synapse/res/templates](synapse/res/templates) into that directory. + +## Synapse SSO Plugins Method Deprecation + +Plugins using the `complete_sso_login` method of +`synapse.module_api.ModuleApi` should update to using the async/await +version `complete_sso_login_async` which includes additional checks. The +non-async version is considered deprecated. + +## Rolling back to v1.12.4 after a failed upgrade + +v1.13.0 includes a lot of large changes. If something problematic +occurs, you may want to roll-back to a previous version of Synapse. +Because v1.13.0 also includes a new database schema version, reverting +that version is also required alongside the generic rollback +instructions mentioned above. In short, to roll back to v1.12.4 you need +to: + +1. Stop the server + +2. Decrease the schema version in the database: + + ```sql + UPDATE schema_version SET version = 57; + ``` + +3. Downgrade Synapse by following the instructions for your + installation method in the "Rolling back to older versions" + section above. + +# Upgrading to v1.12.0 + +This version includes a database update which is run as part of the +upgrade, and which may take some time (several hours in the case of a +large server). Synapse will not respond to HTTP requests while this +update is taking place. + +This is only likely to be a problem in the case of a server which is +participating in many rooms. + +0. As with all upgrades, it is recommended that you have a recent + backup of your database which can be used for recovery in the event + of any problems. + +1. As an initial check to see if you will be affected, you can try + running the following query from the [psql]{.title-ref} or + [sqlite3]{.title-ref} console. It is safe to run it while Synapse is + still running. + + ```sql + SELECT MAX(q.v) FROM ( + SELECT ( + SELECT ej.json AS v + FROM state_events se INNER JOIN event_json ej USING (event_id) + WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key='' + LIMIT 1 + ) FROM rooms WHERE rooms.room_version IS NULL + ) q; + ``` + + This query will take about the same amount of time as the upgrade + process: ie, if it takes 5 minutes, then it is likely that Synapse + will be unresponsive for 5 minutes during the upgrade. + + If you consider an outage of this duration to be acceptable, no + further action is necessary and you can simply start Synapse 1.12.0. + + If you would prefer to reduce the downtime, continue with the steps + below. + +2. The easiest workaround for this issue is to manually create a new + index before upgrading. On PostgreSQL, his can be done as follows: + + ```sql + CREATE INDEX CONCURRENTLY tmp_upgrade_1_12_0_index + ON state_events(room_id) WHERE type = 'm.room.create'; + ``` + + The above query may take some time, but is also safe to run while + Synapse is running. + + We assume that no SQLite users have databases large enough to be + affected. If you *are* affected, you can run a similar query, + omitting the `CONCURRENTLY` keyword. Note however that this + operation may in itself cause Synapse to stop running for some time. + Synapse admins are reminded that [SQLite is not recommended for use + outside a test + environment](https://github.com/matrix-org/synapse/blob/master/README.rst#using-postgresql). + +3. Once the index has been created, the `SELECT` query in step 1 above + should complete quickly. It is therefore safe to upgrade to Synapse + 1.12.0. + +4. Once Synapse 1.12.0 has successfully started and is responding to + HTTP requests, the temporary index can be removed: + + ```sql + DROP INDEX tmp_upgrade_1_12_0_index; + ``` + +# Upgrading to v1.10.0 + +Synapse will now log a warning on start up if used with a PostgreSQL +database that has a non-recommended locale set. + +See [Postgres](../postgres.md) for details. + +# Upgrading to v1.8.0 + +Specifying a `log_file` config option will now cause Synapse to refuse +to start, and should be replaced by with the `log_config` option. +Support for the `log_file` option was removed in v1.3.0 and has since +had no effect. + +# Upgrading to v1.7.0 + +In an attempt to configure Synapse in a privacy preserving way, the +default behaviours of `allow_public_rooms_without_auth` and +`allow_public_rooms_over_federation` have been inverted. This means that +by default, only authenticated users querying the Client/Server API will +be able to query the room directory, and relatedly that the server will +not share room directory information with other servers over federation. + +If your installation does not explicitly set these settings one way or +the other and you want either setting to be `true` then it will +necessary to update your homeserver configuration file accordingly. + +For more details on the surrounding context see our +[explainer](https://matrix.org/blog/2019/11/09/avoiding-unwelcome-visitors-on-private-matrix-servers). + +# Upgrading to v1.5.0 + +This release includes a database migration which may take several +minutes to complete if there are a large number (more than a million or +so) of entries in the `devices` table. This is only likely to a be a +problem on very large installations. + +# Upgrading to v1.4.0 + +## New custom templates + +If you have configured a custom template directory with the +`email.template_dir` option, be aware that there are new templates +regarding registration and threepid management (see below) that must be +included. + +- `registration.html` and `registration.txt` +- `registration_success.html` and `registration_failure.html` +- `add_threepid.html` and `add_threepid.txt` +- `add_threepid_failure.html` and `add_threepid_success.html` + +Synapse will expect these files to exist inside the configured template +directory, and **will fail to start** if they are absent. To view the +default templates, see +[synapse/res/templates](https://github.com/matrix-org/synapse/tree/master/synapse/res/templates). + +## 3pid verification changes + +**Note: As of this release, users will be unable to add phone numbers or +email addresses to their accounts, without changes to the Synapse +configuration. This includes adding an email address during +registration.** + +It is possible for a user to associate an email address or phone number +with their account, for a number of reasons: + +- for use when logging in, as an alternative to the user id. +- in the case of email, as an alternative contact to help with account + recovery. +- in the case of email, to receive notifications of missed messages. + +Before an email address or phone number can be added to a user's +account, or before such an address is used to carry out a +password-reset, Synapse must confirm the operation with the owner of the +email address or phone number. It does this by sending an email or text +giving the user a link or token to confirm receipt. This process is +known as '3pid verification'. ('3pid', or 'threepid', stands for +third-party identifier, and we use it to refer to external identifiers +such as email addresses and phone numbers.) + +Previous versions of Synapse delegated the task of 3pid verification to +an identity server by default. In most cases this server is `vector.im` +or `matrix.org`. + +In Synapse 1.4.0, for security and privacy reasons, the homeserver will +no longer delegate this task to an identity server by default. Instead, +the server administrator will need to explicitly decide how they would +like the verification messages to be sent. + +In the medium term, the `vector.im` and `matrix.org` identity servers +will disable support for delegated 3pid verification entirely. However, +in order to ease the transition, they will retain the capability for a +limited period. Delegated email verification will be disabled on Monday +2nd December 2019 (giving roughly 2 months notice). Disabling delegated +SMS verification will follow some time after that once SMS verification +support lands in Synapse. + +Once delegated 3pid verification support has been disabled in the +`vector.im` and `matrix.org` identity servers, all Synapse versions that +depend on those instances will be unable to verify email and phone +numbers through them. There are no imminent plans to remove delegated +3pid verification from Sydent generally. (Sydent is the identity server +project that backs the `vector.im` and `matrix.org` instances). + +### Email + +Following upgrade, to continue verifying email (e.g. as part of the +registration process), admins can either:- + +- Configure Synapse to use an email server. +- Run or choose an identity server which allows delegated email + verification and delegate to it. + +#### Configure SMTP in Synapse + +To configure an SMTP server for Synapse, modify the configuration +section headed `email`, and be sure to have at least the +`smtp_host, smtp_port` and `notif_from` fields filled out. + +You may also need to set `smtp_user`, `smtp_pass`, and +`require_transport_security`. + +See the [sample configuration file](docs/sample_config.yaml) for more +details on these settings. + +#### Delegate email to an identity server + +Some admins will wish to continue using email verification as part of +the registration process, but will not immediately have an appropriate +SMTP server at hand. + +To this end, we will continue to support email verification delegation +via the `vector.im` and `matrix.org` identity servers for two months. +Support for delegated email verification will be disabled on Monday 2nd +December. + +The `account_threepid_delegates` dictionary defines whether the +homeserver should delegate an external server (typically an [identity +server](https://matrix.org/docs/spec/identity_service/r0.2.1)) to handle +sending confirmation messages via email and SMS. + +So to delegate email verification, in `homeserver.yaml`, set +`account_threepid_delegates.email` to the base URL of an identity +server. For example: + +```yaml +account_threepid_delegates: + email: https://example.com # Delegate email sending to example.com +``` + +Note that `account_threepid_delegates.email` replaces the deprecated +`email.trust_identity_server_for_password_resets`: if +`email.trust_identity_server_for_password_resets` is set to `true`, and +`account_threepid_delegates.email` is not set, then the first entry in +`trusted_third_party_id_servers` will be used as the +`account_threepid_delegate` for email. This is to ensure compatibility +with existing Synapse installs that set up external server handling for +these tasks before v1.4.0. If +`email.trust_identity_server_for_password_resets` is `true` and no +trusted identity server domains are configured, Synapse will report an +error and refuse to start. + +If `email.trust_identity_server_for_password_resets` is `false` or +absent and no `email` delegate is configured in +`account_threepid_delegates`, then Synapse will send email verification +messages itself, using the configured SMTP server (see above). that +type. + +### Phone numbers + +Synapse does not support phone-number verification itself, so the only +way to maintain the ability for users to add phone numbers to their +accounts will be by continuing to delegate phone number verification to +the `matrix.org` and `vector.im` identity servers (or another identity +server that supports SMS sending). + +The `account_threepid_delegates` dictionary defines whether the +homeserver should delegate an external server (typically an [identity +server](https://matrix.org/docs/spec/identity_service/r0.2.1)) to handle +sending confirmation messages via email and SMS. + +So to delegate phone number verification, in `homeserver.yaml`, set +`account_threepid_delegates.msisdn` to the base URL of an identity +server. For example: + +```yaml +account_threepid_delegates: + msisdn: https://example.com # Delegate sms sending to example.com +``` + +The `matrix.org` and `vector.im` identity servers will continue to +support delegated phone number verification via SMS until such time as +it is possible for admins to configure their servers to perform phone +number verification directly. More details will follow in a future +release. + +## Rolling back to v1.3.1 + +If you encounter problems with v1.4.0, it should be possible to roll +back to v1.3.1, subject to the following: + +- The 'room statistics' engine was heavily reworked in this release + (see [#5971](https://github.com/matrix-org/synapse/pull/5971)), + including significant changes to the database schema, which are not + easily reverted. This will cause the room statistics engine to stop + updating when you downgrade. + + The room statistics are essentially unused in v1.3.1 (in future + versions of Synapse, they will be used to populate the room + directory), so there should be no loss of functionality. However, + the statistics engine will write errors to the logs, which can be + avoided by setting the following in `homeserver.yaml`: + + ```yaml + stats: + enabled: false + ``` + + Don't forget to re-enable it when you upgrade again, in preparation + for its use in the room directory! + +# Upgrading to v1.2.0 + +Some counter metrics have been renamed, with the old names deprecated. +See [the metrics +documentation](../metrics-howto.md#renaming-of-metrics--deprecation-of-old-names-in-12) +for details. + +# Upgrading to v1.1.0 + +Synapse v1.1.0 removes support for older Python and PostgreSQL versions, +as outlined in [our deprecation +notice](https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x). + +## Minimum Python Version + +Synapse v1.1.0 has a minimum Python requirement of Python 3.5. Python +3.6 or Python 3.7 are recommended as they have improved internal string +handling, significantly reducing memory usage. + +If you use current versions of the Matrix.org-distributed Debian +packages or Docker images, action is not required. + +If you install Synapse in a Python virtual environment, please see +"Upgrading to v0.34.0" for notes on setting up a new virtualenv under +Python 3. + +## Minimum PostgreSQL Version + +If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 +or above. Please see the [PostgreSQL +documentation](https://www.postgresql.org/docs/11/upgrading.html) for +more details on upgrading your database. + +# Upgrading to v1.0 + +## Validation of TLS certificates + +Synapse v1.0 is the first release to enforce validation of TLS +certificates for the federation API. It is therefore essential that your +certificates are correctly configured. See the +[FAQ](../MSC1711_certificates_FAQ.md) for more information. + +Note, v1.0 installations will also no longer be able to federate with +servers that have not correctly configured their certificates. + +In rare cases, it may be desirable to disable certificate checking: for +example, it might be essential to be able to federate with a given +legacy server in a closed federation. This can be done in one of two +ways:- + +- Configure the global switch `federation_verify_certificates` to + `false`. +- Configure a whitelist of server domains to trust via + `federation_certificate_verification_whitelist`. + +See the [sample configuration file](docs/sample_config.yaml) for more +details on these settings. + +## Email + +When a user requests a password reset, Synapse will send an email to the +user to confirm the request. + +Previous versions of Synapse delegated the job of sending this email to +an identity server. If the identity server was somehow malicious or +became compromised, it would be theoretically possible to hijack an +account through this means. + +Therefore, by default, Synapse v1.0 will send the confirmation email +itself. If Synapse is not configured with an SMTP server, password reset +via email will be disabled. + +To configure an SMTP server for Synapse, modify the configuration +section headed `email`, and be sure to have at least the `smtp_host`, +`smtp_port` and `notif_from` fields filled out. You may also need to set +`smtp_user`, `smtp_pass`, and `require_transport_security`. + +If you are absolutely certain that you wish to continue using an +identity server for password resets, set +`trust_identity_server_for_password_resets` to `true`. + +See the [sample configuration file](docs/sample_config.yaml) for more +details on these settings. + +## New email templates + +Some new templates have been added to the default template directory for the purpose of +the homeserver sending its own password reset emails. If you have configured a +custom `template_dir` in your Synapse config, these files will need to be added. + +`password_reset.html` and `password_reset.txt` are HTML and plain text +templates respectively that contain the contents of what will be emailed +to the user upon attempting to reset their password via email. +`password_reset_success.html` and `password_reset_failure.html` are HTML +files that the content of which (assuming no redirect URL is set) will +be shown to the user after they attempt to click the link in the email +sent to them. + +# Upgrading to v0.99.0 + +Please be aware that, before Synapse v1.0 is released around March 2019, +you will need to replace any self-signed certificates with those +verified by a root CA. Information on how to do so can be found at [the +ACME docs](../ACME.md). + +For more information on configuring TLS certificates see the +[FAQ](../MSC1711_certificates_FAQ.md). + +# Upgrading to v0.34.0 + +1. This release is the first to fully support Python 3. Synapse will + now run on Python versions 3.5, or 3.6 (as well as 2.7). We + recommend switching to Python 3, as it has been shown to give + performance improvements. + + For users who have installed Synapse into a virtualenv, we recommend + doing this by creating a new virtualenv. For example: + + virtualenv -p python3 ~/synapse/env3 + source ~/synapse/env3/bin/activate + pip install matrix-synapse + + You can then start synapse as normal, having activated the new + virtualenv: + + cd ~/synapse + source env3/bin/activate + synctl start + + Users who have installed from distribution packages should see the + relevant package documentation. See below for notes on Debian + packages. + + - When upgrading to Python 3, you **must** make sure that your log + files are configured as UTF-8, by adding `encoding: utf8` to the + `RotatingFileHandler` configuration (if you have one) in your + `<server>.log.config` file. For example, if your `log.config` + file contains: + + handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: homeserver.log + maxBytes: 104857600 + backupCount: 10 + filters: [context] + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + + Then you should update this to be: + + handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: homeserver.log + maxBytes: 104857600 + backupCount: 10 + filters: [context] + encoding: utf8 + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + + There is no need to revert this change if downgrading to + Python 2. + + We are also making available Debian packages which will run Synapse + on Python 3. You can switch to these packages with + `apt-get install matrix-synapse-py3`, however, please read + [debian/NEWS](https://github.com/matrix-org/synapse/blob/release-v0.34.0/debian/NEWS) + before doing so. The existing `matrix-synapse` packages will + continue to use Python 2 for the time being. + +2. This release removes the `riot.im` from the default list of trusted + identity servers. + + If `riot.im` is in your homeserver's list of + `trusted_third_party_id_servers`, you should remove it. It was added + in case a hypothetical future identity server was put there. If you + don't remove it, users may be unable to deactivate their accounts. + +3. This release no longer installs the (unmaintained) Matrix Console + web client as part of the default installation. It is possible to + re-enable it by installing it separately and setting the + `web_client_location` config option, but please consider switching + to another client. + +# Upgrading to v0.33.7 + +This release removes the example email notification templates from +`res/templates` (they are now internal to the python package). This +should only affect you if you (a) deploy your Synapse instance from a +git checkout or a github snapshot URL, and (b) have email notifications +enabled. + +If you have email notifications enabled, you should ensure that +`email.template_dir` is either configured to point at a directory where +you have installed customised templates, or leave it unset to use the +default templates. + +# Upgrading to v0.27.3 + +This release expands the anonymous usage stats sent if the opt-in +`report_stats` configuration is set to `true`. We now capture RSS memory +and cpu use at a very coarse level. This requires administrators to +install the optional `psutil` python module. + +We would appreciate it if you could assist by ensuring this module is +available and `report_stats` is enabled. This will let us see if +performance changes to synapse are having an impact to the general +community. + +# Upgrading to v0.15.0 + +If you want to use the new URL previewing API +(`/_matrix/media/r0/preview_url`) then you have to explicitly enable it +in the config and update your dependencies dependencies. See README.rst +for details. + +# Upgrading to v0.11.0 + +This release includes the option to send anonymous usage stats to +matrix.org, and requires that administrators explictly opt in or out by +setting the `report_stats` option to either `true` or `false`. + +We would really appreciate it if you could help our project out by +reporting anonymized usage statistics from your homeserver. Only very +basic aggregate data (e.g. number of users) will be reported, but it +helps us to track the growth of the Matrix community, and helps us to +make Matrix a success, as well as to convince other networks that they +should peer with us. + +# Upgrading to v0.9.0 + +Application services have had a breaking API change in this version. + +They can no longer register themselves with a home server using the AS +HTTP API. This decision was made because a compromised application +service with free reign to register any regex in effect grants full +read/write access to the home server if a regex of `.*` is used. An +attack where a compromised AS re-registers itself with `.*` was deemed +too big of a security risk to ignore, and so the ability to register +with the HS remotely has been removed. + +It has been replaced by specifying a list of application service +registrations in `homeserver.yaml`: + + app_service_config_files: ["registration-01.yaml", "registration-02.yaml"] + +Where `registration-01.yaml` looks like: + + url: <String> # e.g. "https://my.application.service.com" + as_token: <String> + hs_token: <String> + sender_localpart: <String> # This is a new field which denotes the user_id localpart when using the AS token + namespaces: + users: + - exclusive: <Boolean> + regex: <String> # e.g. "@prefix_.*" + aliases: + - exclusive: <Boolean> + regex: <String> + rooms: + - exclusive: <Boolean> + regex: <String> + +# Upgrading to v0.8.0 + +Servers which use captchas will need to add their public key to: + + static/client/register/register_config.js + + window.matrixRegistrationConfig = { + recaptcha_public_key: "YOUR_PUBLIC_KEY" + }; + +This is required in order to support registration fallback (typically +used on mobile devices). + +# Upgrading to v0.7.0 + +New dependencies are: + +- pydenticon +- simplejson +- syutil +- matrix-angular-sdk + +To pull in these dependencies in a virtual env, run: + + python synapse/python_dependencies.py | xargs -n 1 pip install + +# Upgrading to v0.6.0 + +To pull in new dependencies, run: + + python setup.py develop --user + +This update includes a change to the database schema. To upgrade you +first need to upgrade the database by running: + + python scripts/upgrade_db_to_v0.6.0.py <db> <server_name> <signing_key> + +Where [<db>]{.title-ref} is the location of the database, +[<server_name>]{.title-ref} is the server name as specified in the +synapse configuration, and [<signing_key>]{.title-ref} is the location +of the signing key as specified in the synapse configuration. + +This may take some time to complete. Failures of signatures and content +hashes can safely be ignored. + +# Upgrading to v0.5.1 + +Depending on precisely when you installed v0.5.0 you may have ended up +with a stale release of the reference matrix webclient installed as a +python module. To uninstall it and ensure you are depending on the +latest module, please run: + + $ pip uninstall syweb + +# Upgrading to v0.5.0 + +The webclient has been split out into a seperate repository/pacakage in +this release. Before you restart your homeserver you will need to pull +in the webclient package by running: + + python setup.py develop --user + +This release completely changes the database schema and so requires +upgrading it before starting the new version of the homeserver. + +The script "database-prepare-for-0.5.0.sh" should be used to upgrade +the database. This will save all user information, such as logins and +profiles, but will otherwise purge the database. This includes messages, +which rooms the home server was a member of and room alias mappings. + +If you would like to keep your history, please take a copy of your +database file and ask for help in #matrix:matrix.org. The upgrade +process is, unfortunately, non trivial and requires human intervention +to resolve any resulting conflicts during the upgrade process. + +Before running the command the homeserver should be first completely +shutdown. To run it, simply specify the location of the database, e.g.: + +> ./scripts/database-prepare-for-0.5.0.sh "homeserver.db" + +Once this has successfully completed it will be safe to restart the +homeserver. You may notice that the homeserver takes a few seconds +longer to restart than usual as it reinitializes the database. + +On startup of the new version, users can either rejoin remote rooms +using room aliases or by being reinvited. Alternatively, if any other +homeserver sends a message to a room that the homeserver was previously +in the local HS will automatically rejoin the room. + +# Upgrading to v0.4.0 + +This release needs an updated syutil version. Run: + + python setup.py develop + +You will also need to upgrade your configuration as the signing key +format has changed. Run: + + python -m synapse.app.homeserver --config-path <CONFIG> --generate-config + +# Upgrading to v0.3.0 + +This registration API now closely matches the login API. This introduces +a bit more backwards and forwards between the HS and the client, but +this improves the overall flexibility of the API. You can now GET on +/register to retrieve a list of valid registration flows. Upon choosing +one, they are submitted in the same way as login, e.g: + + { + type: m.login.password, + user: foo, + password: bar + } + +The default HS supports 2 flows, with and without Identity Server email +authentication. Enabling captcha on the HS will add in an extra step to +all flows: `m.login.recaptcha` which must be completed before you can +transition to the next stage. There is a new login type: +`m.login.email.identity` which contains the `threepidCreds` key which +were previously sent in the original register request. For more +information on this, see the specification. + +## Web Client + +The VoIP specification has changed between v0.2.0 and v0.3.0. Users +should refresh any browser tabs to get the latest web client code. Users +on v0.2.0 of the web client will not be able to call those on v0.3.0 and +vice versa. + +# Upgrading to v0.2.0 + +The home server now requires setting up of SSL config before it can run. +To automatically generate default config use: + + $ python synapse/app/homeserver.py \ + --server-name machine.my.domain.name \ + --bind-port 8448 \ + --config-path homeserver.config \ + --generate-config + +This config can be edited if desired, for example to specify a different +SSL certificate to use. Once done you can run the home server using: + + $ python synapse/app/homeserver.py --config-path homeserver.config + +See the README.rst for more information. + +Also note that some config options have been renamed, including: + +- "host" to "server-name" +- "database" to "database-path" +- "port" to "bind-port" and "unsecure-port" + +# Upgrading to v0.0.1 + +This release completely changes the database schema and so requires +upgrading it before starting the new version of the homeserver. + +The script "database-prepare-for-0.0.1.sh" should be used to upgrade +the database. This will save all user information, such as logins and +profiles, but will otherwise purge the database. This includes messages, +which rooms the home server was a member of and room alias mappings. + +Before running the command the homeserver should be first completely +shutdown. To run it, simply specify the location of the database, e.g.: + +> ./scripts/database-prepare-for-0.0.1.sh "homeserver.db" + +Once this has successfully completed it will be safe to restart the +homeserver. You may notice that the homeserver takes a few seconds +longer to restart than usual as it reinitializes the database. + +On startup of the new version, users can either rejoin remote rooms +using room aliases or by being reinvited. Alternatively, if any other +homeserver sends a message to a room that the homeserver was previously +in the local HS will automatically rejoin the room. diff --git a/docs/upgrading/README.md b/docs/upgrading/README.md deleted file mode 100644 index 258e58cf15..0000000000 --- a/docs/upgrading/README.md +++ /dev/null @@ -1,7 +0,0 @@ -<!-- - Include the contents of UPGRADE.rst from the project root without moving it, which may - break links around the internet. Additionally, note that SUMMARY.md is unable to - directly link to content outside of the docs/ directory. So we use this file as a - redirection. ---> -{{#include ../../UPGRADE.rst}} \ No newline at end of file diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index ba060104c3..aca32edc17 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -65,4 +65,4 @@ if [[ -n "$1" ]]; then fi # Run the tests! -go test -v -tags synapse_blacklist,msc2946,msc3083,msc2716 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests +go test -v -tags synapse_blacklist,msc2946,msc3083,msc2716,msc2403 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests diff --git a/scripts-dev/release.py b/scripts-dev/release.py index fc3df9071c..5bfaa4ad2f 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -83,12 +83,6 @@ def run(): if current_version.pre: # If the current version is an RC we don't need to bump any of the # version numbers (other than the RC number). - base_version = "{}.{}.{}".format( - current_version.major, - current_version.minor, - current_version.micro, - ) - if rc: new_version = "{}.{}.{}rc{}".format( current_version.major, @@ -97,49 +91,57 @@ def run(): current_version.pre[1] + 1, ) else: - new_version = base_version + new_version = "{}.{}.{}".format( + current_version.major, + current_version.minor, + current_version.micro, + ) else: - # If this is a new release cycle then we need to know if its a major - # version bump or a hotfix. + # If this is a new release cycle then we need to know if it's a minor + # or a patch version bump. release_type = click.prompt( "Release type", - type=click.Choice(("major", "hotfix")), + type=click.Choice(("minor", "patch")), show_choices=True, - default="major", + default="minor", ) - if release_type == "major": - base_version = new_version = "{}.{}.{}".format( - current_version.major, - current_version.minor + 1, - 0, - ) + if release_type == "minor": if rc: new_version = "{}.{}.{}rc1".format( current_version.major, current_version.minor + 1, 0, ) - + else: + new_version = "{}.{}.{}".format( + current_version.major, + current_version.minor + 1, + 0, + ) else: - base_version = new_version = "{}.{}.{}".format( - current_version.major, - current_version.minor, - current_version.micro + 1, - ) if rc: new_version = "{}.{}.{}rc1".format( current_version.major, current_version.minor, current_version.micro + 1, ) + else: + new_version = "{}.{}.{}".format( + current_version.major, + current_version.minor, + current_version.micro + 1, + ) # Confirm the calculated version is OK. if not click.confirm(f"Create new version: {new_version}?", default=True): click.get_current_context().abort() # Switch to the release branch. - release_branch_name = f"release-v{current_version.major}.{current_version.minor}" + parsed_new_version = version.parse(new_version) + release_branch_name = ( + f"release-v{parsed_new_version.major}.{parsed_new_version.minor}" + ) release_branch = find_ref(repo, release_branch_name) if release_branch: if release_branch.is_remote(): @@ -153,7 +155,7 @@ def run(): # release type. if current_version.is_prerelease: default = release_branch_name - elif release_type == "major": + elif release_type == "minor": default = "develop" else: default = "master" diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 86eb76cbca..2bbaf5557d 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -93,6 +93,7 @@ BOOLEAN_COLUMNS = { "local_media_repository": ["safe_from_quarantine"], "users": ["shadow_banned"], "e2e_fallback_keys_json": ["used"], + "access_tokens": ["used"], } @@ -307,7 +308,8 @@ class Porter(object): information_schema.table_constraints AS tc INNER JOIN information_schema.constraint_column_usage AS ccu USING (table_schema, constraint_name) - WHERE tc.constraint_type = 'FOREIGN KEY'; + WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.table_name != ccu.table_name; """ txn.execute(sql) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index edf1b918eb..f8b068e563 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import pymacaroons from netaddr import IPAddress @@ -31,6 +31,7 @@ from synapse.api.errors import ( from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.appservice import ApplicationService from synapse.events import EventBase +from synapse.events.builder import EventBuilder from synapse.http import get_request_user_agent from synapse.http.site import SynapseRequest from synapse.logging import opentracing as opentracing @@ -245,6 +246,11 @@ class Auth: errcode=Codes.GUEST_ACCESS_FORBIDDEN, ) + # Mark the token as used. This is used to invalidate old refresh + # tokens after some time. + if not user_info.token_used and token_id is not None: + await self.store.mark_access_token_as_used(token_id) + requester = create_requester( user_info.user_id, token_id, @@ -485,7 +491,7 @@ class Auth: def compute_auth_events( self, - event, + event: Union[EventBase, EventBuilder], current_state_ids: StateMap[str], for_verification: bool = False, ) -> List[str]: diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 414e4c019a..8363c2bb0f 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -201,6 +201,12 @@ class EventContentFields: ) +class RoomTypes: + """Understood values of the room_type field of m.room.create events.""" + + SPACE = "m.space" + + class RoomEncryptionAlgorithms: MEGOLM_V1_AES_SHA2 = "m.megolm.v1.aes-sha2" DEFAULT = MEGOLM_V1_AES_SHA2 diff --git a/synapse/config/registration.py b/synapse/config/registration.py index d9dc55a0c3..0ad919b139 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -119,6 +119,27 @@ class RegistrationConfig(Config): session_lifetime = self.parse_duration(session_lifetime) self.session_lifetime = session_lifetime + # The `access_token_lifetime` applies for tokens that can be renewed + # using a refresh token, as per MSC2918. If it is `None`, the refresh + # token mechanism is disabled. + # + # Since it is incompatible with the `session_lifetime` mechanism, it is set to + # `None` by default if a `session_lifetime` is set. + access_token_lifetime = config.get( + "access_token_lifetime", "5m" if session_lifetime is None else None + ) + if access_token_lifetime is not None: + access_token_lifetime = self.parse_duration(access_token_lifetime) + self.access_token_lifetime = access_token_lifetime + + if session_lifetime is not None and access_token_lifetime is not None: + raise ConfigError( + "The refresh token mechanism is incompatible with the " + "`session_lifetime` option. Consider disabling the " + "`session_lifetime` option or disabling the refresh token " + "mechanism by removing the `access_token_lifetime` option." + ) + # The success template used during fallback auth. self.fallback_success_template = self.read_template("auth_success.html") diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 33d7c60241..89bcf81515 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -14,7 +14,7 @@ # limitations under the License. import logging -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple, Union from canonicaljson import encode_canonical_json from signedjson.key import decode_verify_key_bytes @@ -29,6 +29,7 @@ from synapse.api.room_versions import ( RoomVersion, ) from synapse.events import EventBase +from synapse.events.builder import EventBuilder from synapse.types import StateMap, UserID, get_domain_from_id logger = logging.getLogger(__name__) @@ -724,7 +725,7 @@ def get_public_keys(invite_event: EventBase) -> List[Dict[str, Any]]: return public_keys -def auth_types_for_event(event: EventBase) -> Set[Tuple[str, str]]: +def auth_types_for_event(event: Union[EventBase, EventBuilder]) -> Set[Tuple[str, str]]: """Given an event, return a list of (EventType, StateKey) that may be needed to auth the event. The returned list may be a superset of what would actually be required depending on the full state of the room. diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 0cb9c1cc1e..6286ad999a 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -118,7 +118,7 @@ class _EventInternalMetadata: proactively_send = DictProperty("proactively_send") # type: bool redacted = DictProperty("redacted") # type: bool txn_id = DictProperty("txn_id") # type: str - token_id = DictProperty("token_id") # type: str + token_id = DictProperty("token_id") # type: int historical = DictProperty("historical") # type: bool # XXX: These are set by StreamWorkerStore._set_before_and_after. diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 81bf8615b7..fb48ec8541 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import attr from nacl.signing import SigningKey -from synapse.api.auth import Auth from synapse.api.constants import MAX_DEPTH from synapse.api.errors import UnsupportedRoomVersionError from synapse.api.room_versions import ( @@ -34,10 +33,14 @@ from synapse.types import EventID, JsonDict from synapse.util import Clock from synapse.util.stringutils import random_string +if TYPE_CHECKING: + from synapse.api.auth import Auth + from synapse.server import HomeServer + logger = logging.getLogger(__name__) -@attr.s(slots=True, cmp=False, frozen=True) +@attr.s(slots=True, cmp=False, frozen=True, auto_attribs=True) class EventBuilder: """A format independent event builder used to build up the event content before signing the event. @@ -62,31 +65,30 @@ class EventBuilder: _signing_key: The signing key to use to sign the event as the server """ - _state = attr.ib(type=StateHandler) - _auth = attr.ib(type=Auth) - _store = attr.ib(type=DataStore) - _clock = attr.ib(type=Clock) - _hostname = attr.ib(type=str) - _signing_key = attr.ib(type=SigningKey) + _state: StateHandler + _auth: "Auth" + _store: DataStore + _clock: Clock + _hostname: str + _signing_key: SigningKey - room_version = attr.ib(type=RoomVersion) + room_version: RoomVersion - room_id = attr.ib(type=str) - type = attr.ib(type=str) - sender = attr.ib(type=str) + room_id: str + type: str + sender: str - content = attr.ib(default=attr.Factory(dict), type=JsonDict) - unsigned = attr.ib(default=attr.Factory(dict), type=JsonDict) + content: JsonDict = attr.Factory(dict) + unsigned: JsonDict = attr.Factory(dict) # These only exist on a subset of events, so they raise AttributeError if # someone tries to get them when they don't exist. - _state_key = attr.ib(default=None, type=Optional[str]) - _redacts = attr.ib(default=None, type=Optional[str]) - _origin_server_ts = attr.ib(default=None, type=Optional[int]) + _state_key: Optional[str] = None + _redacts: Optional[str] = None + _origin_server_ts: Optional[int] = None - internal_metadata = attr.ib( - default=attr.Factory(lambda: _EventInternalMetadata({})), - type=_EventInternalMetadata, + internal_metadata: _EventInternalMetadata = attr.Factory( + lambda: _EventInternalMetadata({}) ) @property @@ -184,7 +186,7 @@ class EventBuilder: class EventBuilderFactory: - def __init__(self, hs): + def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() self.hostname = hs.hostname self.signing_key = hs.signing_key @@ -193,15 +195,14 @@ class EventBuilderFactory: self.state = hs.get_state_handler() self.auth = hs.get_auth() - def new(self, room_version, key_values): + def new(self, room_version: str, key_values: dict) -> EventBuilder: """Generate an event builder appropriate for the given room version Deprecated: use for_room_version with a RoomVersion object instead Args: - room_version (str): Version of the room that we're creating an event builder - for - key_values (dict): Fields used as the basis of the new event + room_version: Version of the room that we're creating an event builder for + key_values: Fields used as the basis of the new event Returns: EventBuilder @@ -212,13 +213,15 @@ class EventBuilderFactory: raise UnsupportedRoomVersionError() return self.for_room_version(v, key_values) - def for_room_version(self, room_version, key_values): + def for_room_version( + self, room_version: RoomVersion, key_values: dict + ) -> EventBuilder: """Generate an event builder appropriate for the given room version Args: - room_version (synapse.api.room_versions.RoomVersion): + room_version: Version of the room that we're creating an event builder for - key_values (dict): Fields used as the basis of the new event + key_values: Fields used as the basis of the new event Returns: EventBuilder @@ -286,15 +289,15 @@ def create_local_event_from_event_dict( _event_id_counter = 0 -def _create_event_id(clock, hostname): +def _create_event_id(clock: Clock, hostname: str) -> str: """Create a new event ID Args: - clock (Clock) - hostname (str): The server name for the event ID + clock + hostname: The server name for the event ID Returns: - str + The new event ID """ global _event_id_counter diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index c066617b92..2bfe6a3d37 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -89,12 +89,12 @@ class FederationBase: result = await self.spam_checker.check_event_for_spam(pdu) if result: - logger.warning( - "Event contains spam, redacting %s: %s", - pdu.event_id, - pdu.get_pdu_json(), - ) - return prune_event(pdu) + logger.warning("Event contains spam, soft-failing %s", pdu.event_id) + # we redact (to save disk space) as well as soft-failing (to stop + # using the event in prev_events). + redacted_event = prune_event(pdu) + redacted_event.internal_metadata.soft_failed = True + return redacted_event return pdu diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 1d050e54e2..e93b7577fe 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -34,7 +34,7 @@ from twisted.internet import defer from twisted.internet.abstract import isIPAddress from twisted.python import failure -from synapse.api.constants import EduTypes, EventTypes +from synapse.api.constants import EduTypes, EventTypes, Membership from synapse.api.errors import ( AuthError, Codes, @@ -46,6 +46,7 @@ from synapse.api.errors import ( ) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.events import EventBase +from synapse.events.snapshot import EventContext from synapse.federation.federation_base import FederationBase, event_from_pdu_json from synapse.federation.persistence import TransactionActions from synapse.federation.units import Edu, Transaction @@ -368,22 +369,21 @@ class FederationServer(FederationBase): async def process_pdu(pdu: EventBase) -> JsonDict: event_id = pdu.event_id - with pdu_process_time.time(): - with nested_logging_context(event_id): - try: - await self._handle_received_pdu(origin, pdu) - return {} - except FederationError as e: - logger.warning("Error handling PDU %s: %s", event_id, e) - return {"error": str(e)} - except Exception as e: - f = failure.Failure() - logger.error( - "Failed to handle PDU %s", - event_id, - exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore - ) - return {"error": str(e)} + with nested_logging_context(event_id): + try: + await self._handle_received_pdu(origin, pdu) + return {} + except FederationError as e: + logger.warning("Error handling PDU %s: %s", event_id, e) + return {"error": str(e)} + except Exception as e: + f = failure.Failure() + logger.error( + "Failed to handle PDU %s", + event_id, + exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore + ) + return {"error": str(e)} await concurrently_execute( process_pdus_for_room, pdus_by_room.keys(), TRANSACTION_CONCURRENCY_LIMIT @@ -544,26 +544,21 @@ class FederationServer(FederationBase): return {"event": ret_pdu.get_pdu_json(time_now)} async def on_send_join_request( - self, origin: str, content: JsonDict + self, origin: str, content: JsonDict, room_id: str ) -> Dict[str, Any]: - logger.debug("on_send_join_request: content: %s", content) - - assert_params_in_dict(content, ["room_id"]) - room_version = await self.store.get_room_version(content["room_id"]) - pdu = event_from_pdu_json(content, room_version) - - origin_host, _ = parse_server_name(origin) - await self.check_server_matches_acl(origin_host, pdu.room_id) - - logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures) + context = await self._on_send_membership_event( + origin, content, Membership.JOIN, room_id + ) - pdu = await self._check_sigs_and_hash(room_version, pdu) + prev_state_ids = await context.get_prev_state_ids() + state_ids = list(prev_state_ids.values()) + auth_chain = await self.store.get_auth_chain(room_id, state_ids) + state = await self.store.get_events(state_ids) - res_pdus = await self.handler.on_send_join_request(origin, pdu) time_now = self._clock.time_msec() return { - "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]], - "auth_chain": [p.get_pdu_json(time_now) for p in res_pdus["auth_chain"]], + "state": [p.get_pdu_json(time_now) for p in state.values()], + "auth_chain": [p.get_pdu_json(time_now) for p in auth_chain], } async def on_make_leave_request( @@ -578,21 +573,11 @@ class FederationServer(FederationBase): time_now = self._clock.time_msec() return {"event": pdu.get_pdu_json(time_now), "room_version": room_version} - async def on_send_leave_request(self, origin: str, content: JsonDict) -> dict: + async def on_send_leave_request( + self, origin: str, content: JsonDict, room_id: str + ) -> dict: logger.debug("on_send_leave_request: content: %s", content) - - assert_params_in_dict(content, ["room_id"]) - room_version = await self.store.get_room_version(content["room_id"]) - pdu = event_from_pdu_json(content, room_version) - - origin_host, _ = parse_server_name(origin) - await self.check_server_matches_acl(origin_host, pdu.room_id) - - logger.debug("on_send_leave_request: pdu sigs: %s", pdu.signatures) - - pdu = await self._check_sigs_and_hash(room_version, pdu) - - await self.handler.on_send_leave_request(origin, pdu) + await self._on_send_membership_event(origin, content, Membership.LEAVE, room_id) return {} async def on_make_knock_request( @@ -658,39 +643,76 @@ class FederationServer(FederationBase): Returns: The stripped room state. """ - logger.debug("on_send_knock_request: content: %s", content) + event_context = await self._on_send_membership_event( + origin, content, Membership.KNOCK, room_id + ) + + # Retrieve stripped state events from the room and send them back to the remote + # server. This will allow the remote server's clients to display information + # related to the room while the knock request is pending. + stripped_room_state = ( + await self.store.get_stripped_room_state_from_event_context( + event_context, self._room_prejoin_state_types + ) + ) + return {"knock_state_events": stripped_room_state} + + async def _on_send_membership_event( + self, origin: str, content: JsonDict, membership_type: str, room_id: str + ) -> EventContext: + """Handle an on_send_{join,leave,knock} request + + Does some preliminary validation before passing the request on to the + federation handler. + + Args: + origin: The (authenticated) requesting server + content: The body of the send_* request - a complete membership event + membership_type: The expected membership type (join or leave, depending + on the endpoint) + room_id: The room_id from the request, to be validated against the room_id + in the event + + Returns: + The context of the event after inserting it into the room graph. + + Raises: + SynapseError if there is a problem with the request, including things like + the room_id not matching or the event not being authorized. + """ + assert_params_in_dict(content, ["room_id"]) + if content["room_id"] != room_id: + raise SynapseError( + 400, + "Room ID in body does not match that in request path", + Codes.BAD_JSON, + ) room_version = await self.store.get_room_version(room_id) - # Check that this room supports knocking as defined by its room version - if not room_version.msc2403_knocking: + if membership_type == Membership.KNOCK and not room_version.msc2403_knocking: raise SynapseError( 403, "This room version does not support knocking", errcode=Codes.FORBIDDEN, ) - pdu = event_from_pdu_json(content, room_version) + event = event_from_pdu_json(content, room_version) - origin_host, _ = parse_server_name(origin) - await self.check_server_matches_acl(origin_host, pdu.room_id) + if event.type != EventTypes.Member or not event.is_state(): + raise SynapseError(400, "Not an m.room.member event", Codes.BAD_JSON) - logger.debug("on_send_knock_request: pdu sigs: %s", pdu.signatures) + if event.content.get("membership") != membership_type: + raise SynapseError(400, "Not a %s event" % membership_type, Codes.BAD_JSON) - pdu = await self._check_sigs_and_hash(room_version, pdu) + origin_host, _ = parse_server_name(origin) + await self.check_server_matches_acl(origin_host, event.room_id) - # Handle the event, and retrieve the EventContext - event_context = await self.handler.on_send_knock_request(origin, pdu) + logger.debug("_on_send_membership_event: pdu sigs: %s", event.signatures) - # Retrieve stripped state events from the room and send them back to the remote - # server. This will allow the remote server's clients to display information - # related to the room while the knock request is pending. - stripped_room_state = ( - await self.store.get_stripped_room_state_from_event_context( - event_context, self._room_prejoin_state_types - ) - ) - return {"knock_state_events": stripped_room_state} + event = await self._check_sigs_and_hash(room_version, event) + + return await self.handler.on_send_membership_event(origin, event) async def on_event_auth( self, origin: str, room_id: str, event_id: str @@ -909,9 +931,13 @@ class FederationServer(FederationBase): exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore ) - await self.store.remove_received_event_from_staging( + received_ts = await self.store.remove_received_event_from_staging( origin, event.event_id ) + if received_ts is not None: + pdu_process_time.observe( + (self._clock.time_msec() - received_ts) / 1000 + ) # We need to do this check outside the lock to avoid a race between # a new event being inserted by another instance and it attempting diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index bed47f8abd..d37d9565fc 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -15,7 +15,19 @@ import functools import logging import re -from typing import Container, Mapping, Optional, Sequence, Tuple, Type +from typing import ( + Container, + Dict, + List, + Mapping, + Optional, + Sequence, + Tuple, + Type, + Union, +) + +from typing_extensions import Literal import synapse from synapse.api.constants import MAX_GROUP_CATEGORYID_LENGTH, MAX_GROUP_ROLEID_LENGTH @@ -56,15 +68,15 @@ logger = logging.getLogger(__name__) class TransportLayerServer(JsonResource): """Handles incoming federation HTTP requests""" - def __init__(self, hs, servlet_groups=None): + def __init__(self, hs: HomeServer, servlet_groups: Optional[List[str]] = None): """Initialize the TransportLayerServer Will by default register all servlets. For custom behaviour, pass in a list of servlet_groups to register. Args: - hs (synapse.server.HomeServer): homeserver - servlet_groups (list[str], optional): List of servlet groups to register. + hs: homeserver + servlet_groups: List of servlet groups to register. Defaults to ``DEFAULT_SERVLET_GROUPS``. """ self.hs = hs @@ -78,7 +90,7 @@ class TransportLayerServer(JsonResource): self.register_servlets() - def register_servlets(self): + def register_servlets(self) -> None: register_servlets( self.hs, resource=self, @@ -91,14 +103,10 @@ class TransportLayerServer(JsonResource): class AuthenticationError(SynapseError): """There was a problem authenticating the request""" - pass - class NoAuthenticationError(AuthenticationError): """The request had no authentication information""" - pass - class Authenticator: def __init__(self, hs: HomeServer): @@ -410,13 +418,18 @@ class FederationSendServlet(BaseFederationServerServlet): RATELIMIT = False # This is when someone is trying to send us a bunch of data. - async def on_PUT(self, origin, content, query, transaction_id): + async def on_PUT( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + transaction_id: str, + ) -> Tuple[int, JsonDict]: """Called on PUT /send/<transaction_id>/ Args: - request (twisted.web.http.Request): The HTTP request. - transaction_id (str): The transaction_id associated with this - request. This is *not* None. + transaction_id: The transaction_id associated with this request. This + is *not* None. Returns: Tuple of `(code, response)`, where @@ -461,7 +474,13 @@ class FederationEventServlet(BaseFederationServerServlet): PATH = "/event/(?P<event_id>[^/]*)/?" # This is when someone asks for a data item for a given server data_id pair. - async def on_GET(self, origin, content, query, event_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + event_id: str, + ) -> Tuple[int, Union[JsonDict, str]]: return await self.handler.on_pdu_request(origin, event_id) @@ -469,7 +488,13 @@ class FederationStateV1Servlet(BaseFederationServerServlet): PATH = "/state/(?P<room_id>[^/]*)/?" # This is when someone asks for all data for a given room. - async def on_GET(self, origin, content, query, room_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + room_id: str, + ) -> Tuple[int, JsonDict]: return await self.handler.on_room_state_request( origin, room_id, @@ -480,7 +505,13 @@ class FederationStateV1Servlet(BaseFederationServerServlet): class FederationStateIdsServlet(BaseFederationServerServlet): PATH = "/state_ids/(?P<room_id>[^/]*)/?" - async def on_GET(self, origin, content, query, room_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + room_id: str, + ) -> Tuple[int, JsonDict]: return await self.handler.on_state_ids_request( origin, room_id, @@ -491,7 +522,13 @@ class FederationStateIdsServlet(BaseFederationServerServlet): class FederationBackfillServlet(BaseFederationServerServlet): PATH = "/backfill/(?P<room_id>[^/]*)/?" - async def on_GET(self, origin, content, query, room_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + room_id: str, + ) -> Tuple[int, JsonDict]: versions = [x.decode("ascii") for x in query[b"v"]] limit = parse_integer_from_args(query, "limit", None) @@ -505,7 +542,13 @@ class FederationQueryServlet(BaseFederationServerServlet): PATH = "/query/(?P<query_type>[^/]*)" # This is when we receive a server-server Query - async def on_GET(self, origin, content, query, query_type): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + query_type: str, + ) -> Tuple[int, JsonDict]: args = {k.decode("utf8"): v[0].decode("utf-8") for k, v in query.items()} args["origin"] = origin return await self.handler.on_query_request(query_type, args) @@ -514,47 +557,66 @@ class FederationQueryServlet(BaseFederationServerServlet): class FederationMakeJoinServlet(BaseFederationServerServlet): PATH = "/make_join/(?P<room_id>[^/]*)/(?P<user_id>[^/]*)" - async def on_GET(self, origin, _content, query, room_id, user_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + room_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: """ Args: - origin (unicode): The authenticated server_name of the calling server + origin: The authenticated server_name of the calling server - _content (None): (GETs don't have bodies) + content: (GETs don't have bodies) - query (dict[bytes, list[bytes]]): Query params from the request. + query: Query params from the request. - **kwargs (dict[unicode, unicode]): the dict mapping keys to path - components as specified in the path match regexp. + **kwargs: the dict mapping keys to path components as specified in + the path match regexp. Returns: - Tuple[int, object]: (response code, response object) + Tuple of (response code, response object) """ - versions = query.get(b"ver") - if versions is not None: - supported_versions = [v.decode("utf-8") for v in versions] - else: + supported_versions = parse_strings_from_args(query, "ver", encoding="utf-8") + if supported_versions is None: supported_versions = ["1"] - content = await self.handler.on_make_join_request( + result = await self.handler.on_make_join_request( origin, room_id, user_id, supported_versions=supported_versions ) - return 200, content + return 200, result class FederationMakeLeaveServlet(BaseFederationServerServlet): PATH = "/make_leave/(?P<room_id>[^/]*)/(?P<user_id>[^/]*)" - async def on_GET(self, origin, content, query, room_id, user_id): - content = await self.handler.on_make_leave_request(origin, room_id, user_id) - return 200, content + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + room_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: + result = await self.handler.on_make_leave_request(origin, room_id, user_id) + return 200, result class FederationV1SendLeaveServlet(BaseFederationServerServlet): PATH = "/send_leave/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)" - async def on_PUT(self, origin, content, query, room_id, event_id): - content = await self.handler.on_send_leave_request(origin, content) - return 200, (200, content) + async def on_PUT( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + room_id: str, + event_id: str, + ) -> Tuple[int, Tuple[int, JsonDict]]: + result = await self.handler.on_send_leave_request(origin, content, room_id) + return 200, (200, result) class FederationV2SendLeaveServlet(BaseFederationServerServlet): @@ -562,50 +624,84 @@ class FederationV2SendLeaveServlet(BaseFederationServerServlet): PREFIX = FEDERATION_V2_PREFIX - async def on_PUT(self, origin, content, query, room_id, event_id): - content = await self.handler.on_send_leave_request(origin, content) - return 200, content + async def on_PUT( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + room_id: str, + event_id: str, + ) -> Tuple[int, JsonDict]: + result = await self.handler.on_send_leave_request(origin, content, room_id) + return 200, result class FederationMakeKnockServlet(BaseFederationServerServlet): PATH = "/make_knock/(?P<room_id>[^/]*)/(?P<user_id>[^/]*)" - async def on_GET(self, origin, content, query, room_id, user_id): - try: - # Retrieve the room versions the remote homeserver claims to support - supported_versions = parse_strings_from_args(query, "ver", encoding="utf-8") - except KeyError: - raise SynapseError(400, "Missing required query parameter 'ver'") + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + room_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: + # Retrieve the room versions the remote homeserver claims to support + supported_versions = parse_strings_from_args( + query, "ver", required=True, encoding="utf-8" + ) - content = await self.handler.on_make_knock_request( + result = await self.handler.on_make_knock_request( origin, room_id, user_id, supported_versions=supported_versions ) - return 200, content + return 200, result class FederationV1SendKnockServlet(BaseFederationServerServlet): PATH = "/send_knock/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)" - async def on_PUT(self, origin, content, query, room_id, event_id): - content = await self.handler.on_send_knock_request(origin, content, room_id) - return 200, content + async def on_PUT( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + room_id: str, + event_id: str, + ) -> Tuple[int, JsonDict]: + result = await self.handler.on_send_knock_request(origin, content, room_id) + return 200, result class FederationEventAuthServlet(BaseFederationServerServlet): PATH = "/event_auth/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)" - async def on_GET(self, origin, content, query, room_id, event_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + room_id: str, + event_id: str, + ) -> Tuple[int, JsonDict]: return await self.handler.on_event_auth(origin, room_id, event_id) class FederationV1SendJoinServlet(BaseFederationServerServlet): PATH = "/send_join/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)" - async def on_PUT(self, origin, content, query, room_id, event_id): - # TODO(paul): assert that room_id/event_id parsed from path actually + async def on_PUT( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + room_id: str, + event_id: str, + ) -> Tuple[int, Tuple[int, JsonDict]]: + # TODO(paul): assert that event_id parsed from path actually # match those given in content - content = await self.handler.on_send_join_request(origin, content) - return 200, (200, content) + result = await self.handler.on_send_join_request(origin, content, room_id) + return 200, (200, result) class FederationV2SendJoinServlet(BaseFederationServerServlet): @@ -613,28 +709,42 @@ class FederationV2SendJoinServlet(BaseFederationServerServlet): PREFIX = FEDERATION_V2_PREFIX - async def on_PUT(self, origin, content, query, room_id, event_id): - # TODO(paul): assert that room_id/event_id parsed from path actually + async def on_PUT( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + room_id: str, + event_id: str, + ) -> Tuple[int, JsonDict]: + # TODO(paul): assert that event_id parsed from path actually # match those given in content - content = await self.handler.on_send_join_request(origin, content) - return 200, content + result = await self.handler.on_send_join_request(origin, content, room_id) + return 200, result class FederationV1InviteServlet(BaseFederationServerServlet): PATH = "/invite/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)" - async def on_PUT(self, origin, content, query, room_id, event_id): + async def on_PUT( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + room_id: str, + event_id: str, + ) -> Tuple[int, Tuple[int, JsonDict]]: # We don't get a room version, so we have to assume its EITHER v1 or # v2. This is "fine" as the only difference between V1 and V2 is the # state resolution algorithm, and we don't use that for processing # invites - content = await self.handler.on_invite_request( + result = await self.handler.on_invite_request( origin, content, room_version_id=RoomVersions.V1.identifier ) # V1 federation API is defined to return a content of `[200, {...}]` # due to a historical bug. - return 200, (200, content) + return 200, (200, result) class FederationV2InviteServlet(BaseFederationServerServlet): @@ -642,7 +752,14 @@ class FederationV2InviteServlet(BaseFederationServerServlet): PREFIX = FEDERATION_V2_PREFIX - async def on_PUT(self, origin, content, query, room_id, event_id): + async def on_PUT( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + room_id: str, + event_id: str, + ) -> Tuple[int, JsonDict]: # TODO(paul): assert that room_id/event_id parsed from path actually # match those given in content @@ -655,16 +772,22 @@ class FederationV2InviteServlet(BaseFederationServerServlet): event.setdefault("unsigned", {})["invite_room_state"] = invite_room_state - content = await self.handler.on_invite_request( + result = await self.handler.on_invite_request( origin, event, room_version_id=room_version ) - return 200, content + return 200, result class FederationThirdPartyInviteExchangeServlet(BaseFederationServerServlet): PATH = "/exchange_third_party_invite/(?P<room_id>[^/]*)" - async def on_PUT(self, origin, content, query, room_id): + async def on_PUT( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + room_id: str, + ) -> Tuple[int, JsonDict]: await self.handler.on_exchange_third_party_invite_request(content) return 200, {} @@ -672,21 +795,31 @@ class FederationThirdPartyInviteExchangeServlet(BaseFederationServerServlet): class FederationClientKeysQueryServlet(BaseFederationServerServlet): PATH = "/user/keys/query" - async def on_POST(self, origin, content, query): + async def on_POST( + self, origin: str, content: JsonDict, query: Dict[bytes, List[bytes]] + ) -> Tuple[int, JsonDict]: return await self.handler.on_query_client_keys(origin, content) class FederationUserDevicesQueryServlet(BaseFederationServerServlet): PATH = "/user/devices/(?P<user_id>[^/]*)" - async def on_GET(self, origin, content, query, user_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + user_id: str, + ) -> Tuple[int, JsonDict]: return await self.handler.on_query_user_devices(origin, user_id) class FederationClientKeysClaimServlet(BaseFederationServerServlet): PATH = "/user/keys/claim" - async def on_POST(self, origin, content, query): + async def on_POST( + self, origin: str, content: JsonDict, query: Dict[bytes, List[bytes]] + ) -> Tuple[int, JsonDict]: response = await self.handler.on_claim_client_keys(origin, content) return 200, response @@ -695,12 +828,18 @@ class FederationGetMissingEventsServlet(BaseFederationServerServlet): # TODO(paul): Why does this path alone end with "/?" optional? PATH = "/get_missing_events/(?P<room_id>[^/]*)/?" - async def on_POST(self, origin, content, query, room_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + room_id: str, + ) -> Tuple[int, JsonDict]: limit = int(content.get("limit", 10)) earliest_events = content.get("earliest_events", []) latest_events = content.get("latest_events", []) - content = await self.handler.on_get_missing_events( + result = await self.handler.on_get_missing_events( origin, room_id=room_id, earliest_events=earliest_events, @@ -708,7 +847,7 @@ class FederationGetMissingEventsServlet(BaseFederationServerServlet): limit=limit, ) - return 200, content + return 200, result class On3pidBindServlet(BaseFederationServerServlet): @@ -716,7 +855,9 @@ class On3pidBindServlet(BaseFederationServerServlet): REQUIRE_AUTH = False - async def on_POST(self, origin, content, query): + async def on_POST( + self, origin: Optional[str], content: JsonDict, query: Dict[bytes, List[bytes]] + ) -> Tuple[int, JsonDict]: if "invites" in content: last_exception = None for invite in content["invites"]: @@ -762,15 +903,20 @@ class OpenIdUserInfo(BaseFederationServerServlet): REQUIRE_AUTH = False - async def on_GET(self, origin, content, query): - token = query.get(b"access_token", [None])[0] + async def on_GET( + self, + origin: Optional[str], + content: Literal[None], + query: Dict[bytes, List[bytes]], + ) -> Tuple[int, JsonDict]: + token = parse_string_from_args(query, "access_token") if token is None: return ( 401, {"errcode": "M_MISSING_TOKEN", "error": "Access Token required"}, ) - user_id = await self.handler.on_openid_userinfo(token.decode("ascii")) + user_id = await self.handler.on_openid_userinfo(token) if user_id is None: return ( @@ -829,7 +975,9 @@ class PublicRoomList(BaseFederationServlet): self.handler = hs.get_room_list_handler() self.allow_access = allow_access - async def on_GET(self, origin, content, query): + async def on_GET( + self, origin: str, content: Literal[None], query: Dict[bytes, List[bytes]] + ) -> Tuple[int, JsonDict]: if not self.allow_access: raise FederationDeniedError(origin) @@ -858,7 +1006,9 @@ class PublicRoomList(BaseFederationServlet): ) return 200, data - async def on_POST(self, origin, content, query): + async def on_POST( + self, origin: str, content: JsonDict, query: Dict[bytes, List[bytes]] + ) -> Tuple[int, JsonDict]: # This implements MSC2197 (Search Filtering over Federation) if not self.allow_access: raise FederationDeniedError(origin) @@ -904,7 +1054,12 @@ class FederationVersionServlet(BaseFederationServlet): REQUIRE_AUTH = False - async def on_GET(self, origin, content, query): + async def on_GET( + self, + origin: Optional[str], + content: Literal[None], + query: Dict[bytes, List[bytes]], + ) -> Tuple[int, JsonDict]: return ( 200, {"server": {"name": "Synapse", "version": get_version_string(synapse)}}, @@ -933,7 +1088,13 @@ class FederationGroupsProfileServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/profile" - async def on_GET(self, origin, content, query, group_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -942,7 +1103,13 @@ class FederationGroupsProfileServlet(BaseGroupsServerServlet): return 200, new_content - async def on_POST(self, origin, content, query, group_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -957,7 +1124,13 @@ class FederationGroupsProfileServlet(BaseGroupsServerServlet): class FederationGroupsSummaryServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/summary" - async def on_GET(self, origin, content, query, group_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -972,7 +1145,13 @@ class FederationGroupsRoomsServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/rooms" - async def on_GET(self, origin, content, query, group_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -987,7 +1166,14 @@ class FederationGroupsAddRoomsServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/room/(?P<room_id>[^/]*)" - async def on_POST(self, origin, content, query, group_id, room_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + room_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -998,7 +1184,14 @@ class FederationGroupsAddRoomsServlet(BaseGroupsServerServlet): return 200, new_content - async def on_DELETE(self, origin, content, query, group_id, room_id): + async def on_DELETE( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + room_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1018,7 +1211,15 @@ class FederationGroupsAddRoomsConfigServlet(BaseGroupsServerServlet): "/config/(?P<config_key>[^/]*)" ) - async def on_POST(self, origin, content, query, group_id, room_id, config_key): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + room_id: str, + config_key: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1035,7 +1236,13 @@ class FederationGroupsUsersServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/users" - async def on_GET(self, origin, content, query, group_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1050,7 +1257,13 @@ class FederationGroupsInvitedUsersServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/invited_users" - async def on_GET(self, origin, content, query, group_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1067,7 +1280,14 @@ class FederationGroupsInviteServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite" - async def on_POST(self, origin, content, query, group_id, user_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1084,7 +1304,14 @@ class FederationGroupsAcceptInviteServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/accept_invite" - async def on_POST(self, origin, content, query, group_id, user_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: if get_domain_from_id(user_id) != origin: raise SynapseError(403, "user_id doesn't match origin") @@ -1098,7 +1325,14 @@ class FederationGroupsJoinServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/join" - async def on_POST(self, origin, content, query, group_id, user_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: if get_domain_from_id(user_id) != origin: raise SynapseError(403, "user_id doesn't match origin") @@ -1112,7 +1346,14 @@ class FederationGroupsRemoveUserServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove" - async def on_POST(self, origin, content, query, group_id, user_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1146,7 +1387,14 @@ class FederationGroupsLocalInviteServlet(BaseGroupsLocalServlet): PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite" - async def on_POST(self, origin, content, query, group_id, user_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: if get_domain_from_id(group_id) != origin: raise SynapseError(403, "group_id doesn't match origin") @@ -1164,7 +1412,14 @@ class FederationGroupsRemoveLocalUserServlet(BaseGroupsLocalServlet): PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove" - async def on_POST(self, origin, content, query, group_id, user_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + user_id: str, + ) -> Tuple[int, None]: if get_domain_from_id(group_id) != origin: raise SynapseError(403, "user_id doesn't match origin") @@ -1172,11 +1427,9 @@ class FederationGroupsRemoveLocalUserServlet(BaseGroupsLocalServlet): self.handler, GroupsLocalHandler ), "Workers cannot handle group removals." - new_content = await self.handler.user_removed_from_group( - group_id, user_id, content - ) + await self.handler.user_removed_from_group(group_id, user_id, content) - return 200, new_content + return 200, None class FederationGroupsRenewAttestaionServlet(BaseFederationServlet): @@ -1194,7 +1447,14 @@ class FederationGroupsRenewAttestaionServlet(BaseFederationServlet): super().__init__(hs, authenticator, ratelimiter, server_name) self.handler = hs.get_groups_attestation_renewer() - async def on_POST(self, origin, content, query, group_id, user_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: # We don't need to check auth here as we check the attestation signatures new_content = await self.handler.on_renew_attestation( @@ -1218,7 +1478,15 @@ class FederationGroupsSummaryRoomsServlet(BaseGroupsServerServlet): "/rooms/(?P<room_id>[^/]*)" ) - async def on_POST(self, origin, content, query, group_id, category_id, room_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + category_id: str, + room_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1246,7 +1514,15 @@ class FederationGroupsSummaryRoomsServlet(BaseGroupsServerServlet): return 200, resp - async def on_DELETE(self, origin, content, query, group_id, category_id, room_id): + async def on_DELETE( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + category_id: str, + room_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1266,7 +1542,13 @@ class FederationGroupsCategoriesServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/categories/?" - async def on_GET(self, origin, content, query, group_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1281,7 +1563,14 @@ class FederationGroupsCategoryServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/categories/(?P<category_id>[^/]+)" - async def on_GET(self, origin, content, query, group_id, category_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + category_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1292,7 +1581,14 @@ class FederationGroupsCategoryServlet(BaseGroupsServerServlet): return 200, resp - async def on_POST(self, origin, content, query, group_id, category_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + category_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1314,7 +1610,14 @@ class FederationGroupsCategoryServlet(BaseGroupsServerServlet): return 200, resp - async def on_DELETE(self, origin, content, query, group_id, category_id): + async def on_DELETE( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + category_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1334,7 +1637,13 @@ class FederationGroupsRolesServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/roles/?" - async def on_GET(self, origin, content, query, group_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1349,7 +1658,14 @@ class FederationGroupsRoleServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/roles/(?P<role_id>[^/]+)" - async def on_GET(self, origin, content, query, group_id, role_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + role_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1358,7 +1674,14 @@ class FederationGroupsRoleServlet(BaseGroupsServerServlet): return 200, resp - async def on_POST(self, origin, content, query, group_id, role_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + role_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1382,7 +1705,14 @@ class FederationGroupsRoleServlet(BaseGroupsServerServlet): return 200, resp - async def on_DELETE(self, origin, content, query, group_id, role_id): + async def on_DELETE( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + role_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1411,7 +1741,15 @@ class FederationGroupsSummaryUsersServlet(BaseGroupsServerServlet): "/users/(?P<user_id>[^/]*)" ) - async def on_POST(self, origin, content, query, group_id, role_id, user_id): + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + role_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1437,7 +1775,15 @@ class FederationGroupsSummaryUsersServlet(BaseGroupsServerServlet): return 200, resp - async def on_DELETE(self, origin, content, query, group_id, role_id, user_id): + async def on_DELETE( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + group_id: str, + role_id: str, + user_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1457,7 +1803,9 @@ class FederationGroupsBulkPublicisedServlet(BaseGroupsLocalServlet): PATH = "/get_groups_publicised" - async def on_POST(self, origin, content, query): + async def on_POST( + self, origin: str, content: JsonDict, query: Dict[bytes, List[bytes]] + ) -> Tuple[int, JsonDict]: resp = await self.handler.bulk_get_publicised_groups( content["user_ids"], proxy=False ) @@ -1470,7 +1818,13 @@ class FederationGroupsSettingJoinPolicyServlet(BaseGroupsServerServlet): PATH = "/groups/(?P<group_id>[^/]*)/settings/m.join_policy" - async def on_PUT(self, origin, content, query, group_id): + async def on_PUT( + self, + origin: str, + content: JsonDict, + query: Dict[bytes, List[bytes]], + group_id: str, + ) -> Tuple[int, JsonDict]: requester_user_id = parse_string_from_args(query, "requester_user_id") if get_domain_from_id(requester_user_id) != origin: raise SynapseError(403, "requester_user_id doesn't match origin") @@ -1499,7 +1853,7 @@ class FederationSpaceSummaryServlet(BaseFederationServlet): async def on_GET( self, origin: str, - content: JsonDict, + content: Literal[None], query: Mapping[bytes, Sequence[bytes]], room_id: str, ) -> Tuple[int, JsonDict]: @@ -1571,7 +1925,13 @@ class RoomComplexityServlet(BaseFederationServlet): super().__init__(hs, authenticator, ratelimiter, server_name) self._store = self.hs.get_datastore() - async def on_GET(self, origin, content, query, room_id): + async def on_GET( + self, + origin: str, + content: Literal[None], + query: Dict[bytes, List[bytes]], + room_id: str, + ) -> Tuple[int, JsonDict]: is_public = await self._store.is_room_world_readable_or_publicly_joinable( room_id ) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1971e373ed..e2ac595a62 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -30,6 +30,7 @@ from typing import ( Optional, Tuple, Union, + cast, ) import attr @@ -72,6 +73,7 @@ from synapse.util.stringutils import base62_encode from synapse.util.threepids import canonicalise_email if TYPE_CHECKING: + from synapse.rest.client.v1.login import LoginResponse from synapse.server import HomeServer logger = logging.getLogger(__name__) @@ -777,6 +779,108 @@ class AuthHandler(BaseHandler): "params": params, } + async def refresh_token( + self, + refresh_token: str, + valid_until_ms: Optional[int], + ) -> Tuple[str, str]: + """ + Consumes a refresh token and generate both a new access token and a new refresh token from it. + + The consumed refresh token is considered invalid after the first use of the new access token or the new refresh token. + + Args: + refresh_token: The token to consume. + valid_until_ms: The expiration timestamp of the new access token. + + Returns: + A tuple containing the new access token and refresh token + """ + + # Verify the token signature first before looking up the token + if not self._verify_refresh_token(refresh_token): + raise SynapseError(401, "invalid refresh token", Codes.UNKNOWN_TOKEN) + + existing_token = await self.store.lookup_refresh_token(refresh_token) + if existing_token is None: + raise SynapseError(401, "refresh token does not exist", Codes.UNKNOWN_TOKEN) + + if ( + existing_token.has_next_access_token_been_used + or existing_token.has_next_refresh_token_been_refreshed + ): + raise SynapseError( + 403, "refresh token isn't valid anymore", Codes.FORBIDDEN + ) + + ( + new_refresh_token, + new_refresh_token_id, + ) = await self.get_refresh_token_for_user_id( + user_id=existing_token.user_id, device_id=existing_token.device_id + ) + access_token = await self.get_access_token_for_user_id( + user_id=existing_token.user_id, + device_id=existing_token.device_id, + valid_until_ms=valid_until_ms, + refresh_token_id=new_refresh_token_id, + ) + await self.store.replace_refresh_token( + existing_token.token_id, new_refresh_token_id + ) + return access_token, new_refresh_token + + def _verify_refresh_token(self, token: str) -> bool: + """ + Verifies the shape of a refresh token. + + Args: + token: The refresh token to verify + + Returns: + Whether the token has the right shape + """ + parts = token.split("_", maxsplit=4) + if len(parts) != 4: + return False + + type, localpart, rand, crc = parts + + # Refresh tokens are prefixed by "syr_", let's check that + if type != "syr": + return False + + # Check the CRC + base = f"{type}_{localpart}_{rand}" + expected_crc = base62_encode(crc32(base.encode("ascii")), minwidth=6) + if crc != expected_crc: + return False + + return True + + async def get_refresh_token_for_user_id( + self, + user_id: str, + device_id: str, + ) -> Tuple[str, int]: + """ + Creates a new refresh token for the user with the given user ID. + + Args: + user_id: canonical user ID + device_id: the device ID to associate with the token. + + Returns: + The newly created refresh token and its ID in the database + """ + refresh_token = self.generate_refresh_token(UserID.from_string(user_id)) + refresh_token_id = await self.store.add_refresh_token_to_user( + user_id=user_id, + token=refresh_token, + device_id=device_id, + ) + return refresh_token, refresh_token_id + async def get_access_token_for_user_id( self, user_id: str, @@ -784,6 +888,7 @@ class AuthHandler(BaseHandler): valid_until_ms: Optional[int], puppets_user_id: Optional[str] = None, is_appservice_ghost: bool = False, + refresh_token_id: Optional[int] = None, ) -> str: """ Creates a new access token for the user with the given user ID. @@ -801,6 +906,8 @@ class AuthHandler(BaseHandler): valid_until_ms: when the token is valid until. None for no expiry. is_appservice_ghost: Whether the user is an application ghost user + refresh_token_id: the refresh token ID that will be associated with + this access token. Returns: The access token for the user's session. Raises: @@ -836,6 +943,7 @@ class AuthHandler(BaseHandler): device_id=device_id, valid_until_ms=valid_until_ms, puppets_user_id=puppets_user_id, + refresh_token_id=refresh_token_id, ) # the device *should* have been registered before we got here; however, @@ -928,7 +1036,7 @@ class AuthHandler(BaseHandler): self, login_submission: Dict[str, Any], ratelimit: bool = False, - ) -> Tuple[str, Optional[Callable[[Dict[str, str]], Awaitable[None]]]]: + ) -> Tuple[str, Optional[Callable[["LoginResponse"], Awaitable[None]]]]: """Authenticates the user for the /login API Also used by the user-interactive auth flow to validate auth types which don't @@ -1073,7 +1181,7 @@ class AuthHandler(BaseHandler): self, username: str, login_submission: Dict[str, Any], - ) -> Tuple[str, Optional[Callable[[Dict[str, str]], Awaitable[None]]]]: + ) -> Tuple[str, Optional[Callable[["LoginResponse"], Awaitable[None]]]]: """Helper for validate_login Handles login, once we've mapped 3pids onto userids @@ -1151,7 +1259,7 @@ class AuthHandler(BaseHandler): async def check_password_provider_3pid( self, medium: str, address: str, password: str - ) -> Tuple[Optional[str], Optional[Callable[[Dict[str, str]], Awaitable[None]]]]: + ) -> Tuple[Optional[str], Optional[Callable[["LoginResponse"], Awaitable[None]]]]: """Check if a password provider is able to validate a thirdparty login Args: @@ -1215,6 +1323,19 @@ class AuthHandler(BaseHandler): crc = base62_encode(crc32(base.encode("ascii")), minwidth=6) return f"{base}_{crc}" + def generate_refresh_token(self, for_user: UserID) -> str: + """Generates an opaque string, for use as a refresh token""" + + # we use the following format for refresh tokens: + # syr_<base64 local part>_<random string>_<base62 crc check> + + b64local = unpaddedbase64.encode_base64(for_user.localpart.encode("utf-8")) + random_string = stringutils.random_string(20) + base = f"syr_{b64local}_{random_string}" + + crc = base62_encode(crc32(base.encode("ascii")), minwidth=6) + return f"{base}_{crc}" + async def validate_short_term_login_token( self, login_token: str ) -> LoginTokenAttributes: @@ -1563,7 +1684,7 @@ class AuthHandler(BaseHandler): ) respond_with_html(request, 200, html) - async def _sso_login_callback(self, login_result: JsonDict) -> None: + async def _sso_login_callback(self, login_result: "LoginResponse") -> None: """ A login callback which might add additional attributes to the login response. @@ -1577,7 +1698,8 @@ class AuthHandler(BaseHandler): extra_attributes = self._extra_attributes.get(login_result["user_id"]) if extra_attributes: - login_result.update(extra_attributes.extra_attributes) + login_result_dict = cast(Dict[str, Any], login_result) + login_result_dict.update(extra_attributes.extra_attributes) def _expire_sso_extra_attributes(self) -> None: """ diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1b566dbf2d..d929c65131 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1711,80 +1711,6 @@ class FederationHandler(BaseHandler): return event - async def on_send_join_request(self, origin: str, pdu: EventBase) -> JsonDict: - """We have received a join event for a room. Fully process it and - respond with the current state and auth chains. - """ - event = pdu - - logger.debug( - "on_send_join_request from %s: Got event: %s, signatures: %s", - origin, - event.event_id, - event.signatures, - ) - - if get_domain_from_id(event.sender) != origin: - logger.info( - "Got /send_join request for user %r from different origin %s", - event.sender, - origin, - ) - raise SynapseError(403, "User not from origin", Codes.FORBIDDEN) - - event.internal_metadata.outlier = False - # Send this event on behalf of the origin server. - # - # The reasons we have the destination server rather than the origin - # server send it are slightly mysterious: the origin server should have - # all the necessary state once it gets the response to the send_join, - # so it could send the event itself if it wanted to. It may be that - # doing it this way reduces failure modes, or avoids certain attacks - # where a new server selectively tells a subset of the federation that - # it has joined. - # - # The fact is that, as of the current writing, Synapse doesn't send out - # the join event over federation after joining, and changing it now - # would introduce the danger of backwards-compatibility problems. - event.internal_metadata.send_on_behalf_of = origin - - # Calculate the event context. - context = await self.state_handler.compute_event_context(event) - - # Get the state before the new event. - prev_state_ids = await context.get_prev_state_ids() - - # Check if the user is already in the room or invited to the room. - user_id = event.state_key - prev_member_event_id = prev_state_ids.get((EventTypes.Member, user_id), None) - prev_member_event = None - if prev_member_event_id: - prev_member_event = await self.store.get_event(prev_member_event_id) - - # Check if the member should be allowed access via membership in a space. - await self._event_auth_handler.check_restricted_join_rules( - prev_state_ids, - event.room_version, - user_id, - prev_member_event, - ) - - # Persist the event. - await self._auth_and_persist_event(origin, event, context) - - logger.debug( - "on_send_join_request: After _auth_and_persist_event: %s, sigs: %s", - event.event_id, - event.signatures, - ) - - state_ids = list(prev_state_ids.values()) - auth_chain = await self.store.get_auth_chain(event.room_id, state_ids) - - state = await self.store.get_events(list(prev_state_ids.values())) - - return {"state": list(state.values()), "auth_chain": auth_chain} - async def on_invite_request( self, origin: str, event: EventBase, room_version: RoomVersion ) -> EventBase: @@ -1960,37 +1886,6 @@ class FederationHandler(BaseHandler): return event - async def on_send_leave_request(self, origin: str, pdu: EventBase) -> None: - """We have received a leave event for a room. Fully process it.""" - event = pdu - - logger.debug( - "on_send_leave_request: Got event: %s, signatures: %s", - event.event_id, - event.signatures, - ) - - if get_domain_from_id(event.sender) != origin: - logger.info( - "Got /send_leave request for user %r from different origin %s", - event.sender, - origin, - ) - raise SynapseError(403, "User not from origin", Codes.FORBIDDEN) - - event.internal_metadata.outlier = False - - context = await self.state_handler.compute_event_context(event) - await self._auth_and_persist_event(origin, event, context) - - logger.debug( - "on_send_leave_request: After _auth_and_persist_event: %s, sigs: %s", - event.event_id, - event.signatures, - ) - - return None - @log_function async def on_make_knock_request( self, origin: str, room_id: str, user_id: str @@ -2054,51 +1949,115 @@ class FederationHandler(BaseHandler): return event @log_function - async def on_send_knock_request( + async def on_send_membership_event( self, origin: str, event: EventBase ) -> EventContext: """ - We have received a knock event for a room. Verify that event and send it into the room - on the knocking homeserver's behalf. + We have received a join/leave/knock event for a room via send_join/leave/knock. + + Verify that event and send it into the room on the remote homeserver's behalf. + + This is quite similar to on_receive_pdu, with the following principal + differences: + * only membership events are permitted (and only events with + sender==state_key -- ie, no kicks or bans) + * *We* send out the event on behalf of the remote server. + * We enforce the membership restrictions of restricted rooms. + * Rejected events result in an exception rather than being stored. + + There are also other differences, however it is not clear if these are by + design or omission. In particular, we do not attempt to backfill any missing + prev_events. Args: - origin: The remote homeserver of the knocking user. - event: The knocking member event that has been signed by the remote homeserver. + origin: The homeserver of the remote (joining/invited/knocking) user. + event: The member event that has been signed by the remote homeserver. Returns: The context of the event after inserting it into the room graph. + + Raises: + SynapseError if the event is not accepted into the room """ logger.debug( - "on_send_knock_request: Got event: %s, signatures: %s", + "on_send_membership_event: Got event: %s, signatures: %s", event.event_id, event.signatures, ) if get_domain_from_id(event.sender) != origin: logger.info( - "Got /send_knock request for user %r from different origin %s", + "Got send_membership request for user %r from different origin %s", event.sender, origin, ) raise SynapseError(403, "User not from origin", Codes.FORBIDDEN) - event.internal_metadata.outlier = False + if event.sender != event.state_key: + raise SynapseError(400, "state_key and sender must match", Codes.BAD_JSON) - context = await self.state_handler.compute_event_context(event) + assert not event.internal_metadata.outlier - event_allowed = await self.third_party_event_rules.check_event_allowed( - event, context - ) - if not event_allowed: - logger.info("Sending of knock %s forbidden by third-party rules", event) + # Send this event on behalf of the other server. + # + # The remote server isn't a full participant in the room at this point, so + # may not have an up-to-date list of the other homeservers participating in + # the room, so we send it on their behalf. + event.internal_metadata.send_on_behalf_of = origin + + context = await self.state_handler.compute_event_context(event) + context = await self._check_event_auth(origin, event, context) + if context.rejected: raise SynapseError( - 403, "This event is not allowed in this context", Codes.FORBIDDEN + 403, f"{event.membership} event was rejected", Codes.FORBIDDEN ) - await self._auth_and_persist_event(origin, event, context) + # for joins, we need to check the restrictions of restricted rooms + if event.membership == Membership.JOIN: + await self._check_join_restrictions(context, event) + # for knock events, we run the third-party event rules. It's not entirely clear + # why we don't do this for other sorts of membership events. + if event.membership == Membership.KNOCK: + event_allowed = await self.third_party_event_rules.check_event_allowed( + event, context + ) + if not event_allowed: + logger.info("Sending of knock %s forbidden by third-party rules", event) + raise SynapseError( + 403, "This event is not allowed in this context", Codes.FORBIDDEN + ) + + # all looks good, we can persist the event. + await self._run_push_actions_and_persist_event(event, context) return context + async def _check_join_restrictions( + self, context: EventContext, event: EventBase + ) -> None: + """Check that restrictions in restricted join rules are matched + + Called when we receive a join event via send_join. + + Raises an auth error if the restrictions are not matched. + """ + prev_state_ids = await context.get_prev_state_ids() + + # Check if the user is already in the room or invited to the room. + user_id = event.state_key + prev_member_event_id = prev_state_ids.get((EventTypes.Member, user_id), None) + prev_member_event = None + if prev_member_event_id: + prev_member_event = await self.store.get_event(prev_member_event_id) + + # Check if the member should be allowed access via membership in a space. + await self._event_auth_handler.check_restricted_join_rules( + prev_state_ids, + event.room_version, + user_id, + prev_member_event, + ) + async def get_state_for_pdu(self, room_id: str, event_id: str) -> List[EventBase]: """Returns the state at the event. i.e. not including said event.""" @@ -2240,6 +2199,18 @@ class FederationHandler(BaseHandler): backfilled=backfilled, ) + await self._run_push_actions_and_persist_event(event, context, backfilled) + + async def _run_push_actions_and_persist_event( + self, event: EventBase, context: EventContext, backfilled: bool = False + ): + """Run the push actions for a received event, and persist it. + + Args: + event: The event itself. + context: The event context. + backfilled: True if the event was backfilled. + """ try: if ( not event.internal_metadata.is_outlier() @@ -2553,9 +2524,9 @@ class FederationHandler(BaseHandler): origin: str, event: EventBase, context: EventContext, - state: Optional[Iterable[EventBase]], - auth_events: Optional[MutableStateMap[EventBase]], - backfilled: bool, + state: Optional[Iterable[EventBase]] = None, + auth_events: Optional[MutableStateMap[EventBase]] = None, + backfilled: bool = False, ) -> EventContext: """ Checks whether an event should be rejected (for failing auth checks). diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index db12abd59d..364c5cd2d3 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -509,6 +509,8 @@ class EventCreationHandler: Should normally be left as None, which will cause them to be calculated based on the room state at the prev_events. + If non-None, prev_event_ids must also be provided. + require_consent: Whether to check if the requester has consented to the privacy policy. @@ -581,6 +583,9 @@ class EventCreationHandler: # Strip down the auth_event_ids to only what we need to auth the event. # For example, we don't need extra m.room.member that don't match event.sender if auth_event_ids is not None: + # If auth events are provided, prev events must be also. + assert prev_event_ids is not None + temp_event = await builder.build( prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids, @@ -784,6 +789,8 @@ class EventCreationHandler: The event ids to use as the auth_events for the new event. Should normally be left as None, which will cause them to be calculated based on the room state at the prev_events. + + If non-None, prev_event_ids must also be provided. ratelimit: Whether to rate limit this send. txn_id: The transaction ID. ignore_shadow_ban: True if shadow-banned users should be allowed to diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index ca1ed6a5c0..26ef016179 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -15,9 +15,10 @@ """Contains functions for registering clients.""" import logging -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple +from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple from prometheus_client import Counter +from typing_extensions import TypedDict from synapse import types from synapse.api.constants import MAX_USERID_LENGTH, EventTypes, JoinRules, LoginType @@ -54,6 +55,16 @@ login_counter = Counter( ["guest", "auth_provider"], ) +LoginDict = TypedDict( + "LoginDict", + { + "device_id": str, + "access_token": str, + "valid_until_ms": Optional[int], + "refresh_token": Optional[str], + }, +) + class RegistrationHandler(BaseHandler): def __init__(self, hs: "HomeServer"): @@ -85,6 +96,7 @@ class RegistrationHandler(BaseHandler): self.pusher_pool = hs.get_pusherpool() self.session_lifetime = hs.config.session_lifetime + self.access_token_lifetime = hs.config.access_token_lifetime async def check_username( self, @@ -386,11 +398,32 @@ class RegistrationHandler(BaseHandler): room_alias = RoomAlias.from_string(r) if self.hs.hostname != room_alias.domain: - logger.warning( - "Cannot create room alias %s, " - "it does not match server domain", + # If the alias is remote, try to join the room. This might fail + # because the room might be invite only, but we don't have any local + # user in the room to invite this one with, so at this point that's + # the best we can do. + logger.info( + "Cannot automatically create room with alias %s as it isn't" + " local, trying to join the room instead", r, ) + + ( + room, + remote_room_hosts, + ) = await room_member_handler.lookup_room_alias(room_alias) + room_id = room.to_string() + + await room_member_handler.update_membership( + requester=create_requester( + user_id, authenticated_entity=self._server_name + ), + target=UserID.from_string(user_id), + room_id=room_id, + remote_room_hosts=remote_room_hosts, + action="join", + ratelimit=False, + ) else: # A shallow copy is OK here since the only key that is # modified is room_alias_name. @@ -448,22 +481,32 @@ class RegistrationHandler(BaseHandler): ) # Calculate whether the room requires an invite or can be - # joined directly. Note that unless a join rule of public exists, - # it is treated as requiring an invite. - requires_invite = True - - state = await self.store.get_filtered_current_state_ids( - room_id, StateFilter.from_types([(EventTypes.JoinRules, "")]) + # joined directly. By default, we consider the room as requiring an + # invite if the homeserver is in the room (unless told otherwise by the + # join rules). Otherwise we consider it as being joinable, at the risk of + # failing to join, but in this case there's little more we can do since + # we don't have a local user in the room to craft up an invite with. + requires_invite = await self.store.is_host_joined( + room_id, + self.server_name, ) - event_id = state.get((EventTypes.JoinRules, "")) - if event_id: - join_rules_event = await self.store.get_event( - event_id, allow_none=True + if requires_invite: + # If the server is in the room, check if the room is public. + state = await self.store.get_filtered_current_state_ids( + room_id, StateFilter.from_types([(EventTypes.JoinRules, "")]) ) - if join_rules_event: - join_rule = join_rules_event.content.get("join_rule", None) - requires_invite = join_rule and join_rule != JoinRules.PUBLIC + + event_id = state.get((EventTypes.JoinRules, "")) + if event_id: + join_rules_event = await self.store.get_event( + event_id, allow_none=True + ) + if join_rules_event: + join_rule = join_rules_event.content.get("join_rule", None) + requires_invite = ( + join_rule and join_rule != JoinRules.PUBLIC + ) # Send the invite, if necessary. if requires_invite: @@ -665,7 +708,8 @@ class RegistrationHandler(BaseHandler): is_guest: bool = False, is_appservice_ghost: bool = False, auth_provider_id: Optional[str] = None, - ) -> Tuple[str, str]: + should_issue_refresh_token: bool = False, + ) -> Tuple[str, str, Optional[int], Optional[str]]: """Register a device for a user and generate an access token. The access token will be limited by the homeserver's session_lifetime config. @@ -677,8 +721,9 @@ class RegistrationHandler(BaseHandler): is_guest: Whether this is a guest account auth_provider_id: The SSO IdP the user used, if any (just used for the prometheus metrics). + should_issue_refresh_token: Whether it should also issue a refresh token Returns: - Tuple of device ID and access token + Tuple of device ID, access token, access token expiration time and refresh token """ res = await self._register_device_client( user_id=user_id, @@ -686,6 +731,7 @@ class RegistrationHandler(BaseHandler): initial_display_name=initial_display_name, is_guest=is_guest, is_appservice_ghost=is_appservice_ghost, + should_issue_refresh_token=should_issue_refresh_token, ) login_counter.labels( @@ -693,7 +739,12 @@ class RegistrationHandler(BaseHandler): auth_provider=(auth_provider_id or ""), ).inc() - return res["device_id"], res["access_token"] + return ( + res["device_id"], + res["access_token"], + res["valid_until_ms"], + res["refresh_token"], + ) async def register_device_inner( self, @@ -702,7 +753,8 @@ class RegistrationHandler(BaseHandler): initial_display_name: Optional[str], is_guest: bool = False, is_appservice_ghost: bool = False, - ) -> Dict[str, str]: + should_issue_refresh_token: bool = False, + ) -> LoginDict: """Helper for register_device Does the bits that need doing on the main process. Not for use outside this @@ -717,6 +769,9 @@ class RegistrationHandler(BaseHandler): ) valid_until_ms = self.clock.time_msec() + self.session_lifetime + refresh_token = None + refresh_token_id = None + registered_device_id = await self.device_handler.check_device_registered( user_id, device_id, initial_display_name ) @@ -724,14 +779,30 @@ class RegistrationHandler(BaseHandler): assert valid_until_ms is None access_token = self.macaroon_gen.generate_guest_access_token(user_id) else: + if should_issue_refresh_token: + ( + refresh_token, + refresh_token_id, + ) = await self._auth_handler.get_refresh_token_for_user_id( + user_id, + device_id=registered_device_id, + ) + valid_until_ms = self.clock.time_msec() + self.access_token_lifetime + access_token = await self._auth_handler.get_access_token_for_user_id( user_id, device_id=registered_device_id, valid_until_ms=valid_until_ms, is_appservice_ghost=is_appservice_ghost, + refresh_token_id=refresh_token_id, ) - return {"device_id": registered_device_id, "access_token": access_token} + return { + "device_id": registered_device_id, + "access_token": access_token, + "valid_until_ms": valid_until_ms, + "refresh_token": refresh_token, + } async def post_registration_actions( self, user_id: str, auth_result: dict, access_token: Optional[str] diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py index 17fc47ce16..266f369883 100644 --- a/synapse/handlers/space_summary.py +++ b/synapse/handlers/space_summary.py @@ -25,6 +25,7 @@ from synapse.api.constants import ( EventTypes, HistoryVisibility, Membership, + RoomTypes, ) from synapse.events import EventBase from synapse.events.utils import format_event_for_client_v2 @@ -318,7 +319,8 @@ class SpaceSummaryHandler: Returns: A tuple of: - An iterable of a single value of the room. + The room information, if the room should be returned to the + user. None, otherwise. An iterable of the sorted children events. This may be limited to a maximum size or may include all children. @@ -328,7 +330,11 @@ class SpaceSummaryHandler: room_entry = await self._build_room_entry(room_id) - # look for child rooms/spaces. + # If the room is not a space, return just the room information. + if room_entry.get("room_type") != RoomTypes.SPACE: + return room_entry, () + + # Otherwise, look for child rooms/spaces. child_events = await self._get_child_events(room_id) if suggested_only: @@ -348,6 +354,7 @@ class SpaceSummaryHandler: event_format=format_event_for_client_v2, ) ) + return room_entry, events_result async def _summarize_remote_room( diff --git a/synapse/http/server.py b/synapse/http/server.py index 845651e606..efbc6d5b25 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -728,7 +728,7 @@ def set_cors_headers(request: Request): ) request.setHeader( b"Access-Control-Allow-Headers", - b"Origin, X-Requested-With, Content-Type, Accept, Authorization, Date", + b"X-Requested-With, Content-Type, Authorization, Date", ) diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index fda8da21b7..6ba2ce1e53 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -113,8 +113,18 @@ def parse_boolean_from_args(args, name, default=None, required=False): def parse_bytes_from_args( args: Dict[bytes, List[bytes]], name: str, + default: Optional[bytes] = None, +) -> Optional[bytes]: + ... + + +@overload +def parse_bytes_from_args( + args: Dict[bytes, List[bytes]], + name: str, default: Literal[None] = None, - required: Literal[True] = True, + *, + required: Literal[True], ) -> bytes: ... @@ -197,7 +207,12 @@ def parse_string( """ args = request.args # type: Dict[bytes, List[bytes]] # type: ignore return parse_string_from_args( - args, name, default, required, allowed_values, encoding + args, + name, + default, + required=required, + allowed_values=allowed_values, + encoding=encoding, ) @@ -227,7 +242,20 @@ def parse_strings_from_args( args: Dict[bytes, List[bytes]], name: str, default: Optional[List[str]] = None, - required: Literal[True] = True, + *, + allowed_values: Optional[Iterable[str]] = None, + encoding: str = "ascii", +) -> Optional[List[str]]: + ... + + +@overload +def parse_strings_from_args( + args: Dict[bytes, List[bytes]], + name: str, + default: Optional[List[str]] = None, + *, + required: Literal[True], allowed_values: Optional[Iterable[str]] = None, encoding: str = "ascii", ) -> List[str]: @@ -239,6 +267,7 @@ def parse_strings_from_args( args: Dict[bytes, List[bytes]], name: str, default: Optional[List[str]] = None, + *, required: bool = False, allowed_values: Optional[Iterable[str]] = None, encoding: str = "ascii", @@ -299,7 +328,20 @@ def parse_string_from_args( args: Dict[bytes, List[bytes]], name: str, default: Optional[str] = None, - required: Literal[True] = True, + *, + allowed_values: Optional[Iterable[str]] = None, + encoding: str = "ascii", +) -> Optional[str]: + ... + + +@overload +def parse_string_from_args( + args: Dict[bytes, List[bytes]], + name: str, + default: Optional[str] = None, + *, + required: Literal[True], allowed_values: Optional[Iterable[str]] = None, encoding: str = "ascii", ) -> str: diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 58b255eb1b..721c45abac 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -168,7 +168,7 @@ class ModuleApi: "Using deprecated ModuleApi.register which creates a dummy user device." ) user_id = yield self.register_user(localpart, displayname, emails or []) - _, access_token = yield self.register_device(user_id) + _, access_token, _, _ = yield self.register_device(user_id) return user_id, access_token def register_user( diff --git a/synapse/replication/http/login.py b/synapse/replication/http/login.py index c2e8c00293..550bd5c95f 100644 --- a/synapse/replication/http/login.py +++ b/synapse/replication/http/login.py @@ -36,20 +36,29 @@ class RegisterDeviceReplicationServlet(ReplicationEndpoint): @staticmethod async def _serialize_payload( - user_id, device_id, initial_display_name, is_guest, is_appservice_ghost + user_id, + device_id, + initial_display_name, + is_guest, + is_appservice_ghost, + should_issue_refresh_token, ): """ Args: + user_id (int) device_id (str|None): Device ID to use, if None a new one is generated. initial_display_name (str|None) is_guest (bool) + is_appservice_ghost (bool) + should_issue_refresh_token (bool) """ return { "device_id": device_id, "initial_display_name": initial_display_name, "is_guest": is_guest, "is_appservice_ghost": is_appservice_ghost, + "should_issue_refresh_token": should_issue_refresh_token, } async def _handle_request(self, request, user_id): @@ -59,6 +68,7 @@ class RegisterDeviceReplicationServlet(ReplicationEndpoint): initial_display_name = content["initial_display_name"] is_guest = content["is_guest"] is_appservice_ghost = content["is_appservice_ghost"] + should_issue_refresh_token = content["should_issue_refresh_token"] res = await self.registration_handler.register_device_inner( user_id, @@ -66,6 +76,7 @@ class RegisterDeviceReplicationServlet(ReplicationEndpoint): initial_display_name, is_guest, is_appservice_ghost=is_appservice_ghost, + should_issue_refresh_token=should_issue_refresh_token, ) return 200, res diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index f6be5f1020..cbcb60fe31 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -14,7 +14,9 @@ import logging import re -from typing import TYPE_CHECKING, Awaitable, Callable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional + +from typing_extensions import TypedDict from synapse.api.errors import Codes, LoginError, SynapseError from synapse.api.ratelimiting import Ratelimiter @@ -25,6 +27,8 @@ from synapse.http import get_request_uri from synapse.http.server import HttpServer, finish_request from synapse.http.servlet import ( RestServlet, + assert_params_in_dict, + parse_boolean, parse_bytes_from_args, parse_json_object_from_request, parse_string, @@ -40,6 +44,21 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +LoginResponse = TypedDict( + "LoginResponse", + { + "user_id": str, + "access_token": str, + "home_server": str, + "expires_in_ms": Optional[int], + "refresh_token": Optional[str], + "device_id": str, + "well_known": Optional[Dict[str, Any]], + }, + total=False, +) + + class LoginRestServlet(RestServlet): PATTERNS = client_patterns("/login$", v1=True) CAS_TYPE = "m.login.cas" @@ -48,6 +67,7 @@ class LoginRestServlet(RestServlet): JWT_TYPE = "org.matrix.login.jwt" JWT_TYPE_DEPRECATED = "m.login.jwt" APPSERVICE_TYPE = "uk.half-shot.msc2778.login.application_service" + REFRESH_TOKEN_PARAM = "org.matrix.msc2918.refresh_token" def __init__(self, hs: "HomeServer"): super().__init__() @@ -65,9 +85,12 @@ class LoginRestServlet(RestServlet): self.cas_enabled = hs.config.cas_enabled self.oidc_enabled = hs.config.oidc_enabled self._msc2858_enabled = hs.config.experimental.msc2858_enabled + self._msc2918_enabled = hs.config.access_token_lifetime is not None self.auth = hs.get_auth() + self.clock = hs.get_clock() + self.auth_handler = self.hs.get_auth_handler() self.registration_handler = hs.get_registration_handler() self._sso_handler = hs.get_sso_handler() @@ -138,6 +161,15 @@ class LoginRestServlet(RestServlet): async def on_POST(self, request: SynapseRequest): login_submission = parse_json_object_from_request(request) + if self._msc2918_enabled: + # Check if this login should also issue a refresh token, as per + # MSC2918 + should_issue_refresh_token = parse_boolean( + request, name=LoginRestServlet.REFRESH_TOKEN_PARAM, default=False + ) + else: + should_issue_refresh_token = False + try: if login_submission["type"] == LoginRestServlet.APPSERVICE_TYPE: appservice = self.auth.get_appservice_by_req(request) @@ -147,19 +179,32 @@ class LoginRestServlet(RestServlet): None, request.getClientIP() ) - result = await self._do_appservice_login(login_submission, appservice) + result = await self._do_appservice_login( + login_submission, + appservice, + should_issue_refresh_token=should_issue_refresh_token, + ) elif self.jwt_enabled and ( login_submission["type"] == LoginRestServlet.JWT_TYPE or login_submission["type"] == LoginRestServlet.JWT_TYPE_DEPRECATED ): await self._address_ratelimiter.ratelimit(None, request.getClientIP()) - result = await self._do_jwt_login(login_submission) + result = await self._do_jwt_login( + login_submission, + should_issue_refresh_token=should_issue_refresh_token, + ) elif login_submission["type"] == LoginRestServlet.TOKEN_TYPE: await self._address_ratelimiter.ratelimit(None, request.getClientIP()) - result = await self._do_token_login(login_submission) + result = await self._do_token_login( + login_submission, + should_issue_refresh_token=should_issue_refresh_token, + ) else: await self._address_ratelimiter.ratelimit(None, request.getClientIP()) - result = await self._do_other_login(login_submission) + result = await self._do_other_login( + login_submission, + should_issue_refresh_token=should_issue_refresh_token, + ) except KeyError: raise SynapseError(400, "Missing JSON keys.") @@ -169,7 +214,10 @@ class LoginRestServlet(RestServlet): return 200, result async def _do_appservice_login( - self, login_submission: JsonDict, appservice: ApplicationService + self, + login_submission: JsonDict, + appservice: ApplicationService, + should_issue_refresh_token: bool = False, ): identifier = login_submission.get("identifier") logger.info("Got appservice login request with identifier: %r", identifier) @@ -198,14 +246,21 @@ class LoginRestServlet(RestServlet): raise LoginError(403, "Invalid access_token", errcode=Codes.FORBIDDEN) return await self._complete_login( - qualified_user_id, login_submission, ratelimit=appservice.is_rate_limited() + qualified_user_id, + login_submission, + ratelimit=appservice.is_rate_limited(), + should_issue_refresh_token=should_issue_refresh_token, ) - async def _do_other_login(self, login_submission: JsonDict) -> Dict[str, str]: + async def _do_other_login( + self, login_submission: JsonDict, should_issue_refresh_token: bool = False + ) -> LoginResponse: """Handle non-token/saml/jwt logins Args: login_submission: + should_issue_refresh_token: True if this login should issue + a refresh token alongside the access token. Returns: HTTP response @@ -224,7 +279,10 @@ class LoginRestServlet(RestServlet): login_submission, ratelimit=True ) result = await self._complete_login( - canonical_user_id, login_submission, callback + canonical_user_id, + login_submission, + callback, + should_issue_refresh_token=should_issue_refresh_token, ) return result @@ -232,11 +290,12 @@ class LoginRestServlet(RestServlet): self, user_id: str, login_submission: JsonDict, - callback: Optional[Callable[[Dict[str, str]], Awaitable[None]]] = None, + callback: Optional[Callable[[LoginResponse], Awaitable[None]]] = None, create_non_existent_users: bool = False, ratelimit: bool = True, auth_provider_id: Optional[str] = None, - ) -> Dict[str, str]: + should_issue_refresh_token: bool = False, + ) -> LoginResponse: """Called when we've successfully authed the user and now need to actually login them in (e.g. create devices). This gets called on all successful logins. @@ -253,6 +312,8 @@ class LoginRestServlet(RestServlet): ratelimit: Whether to ratelimit the login request. auth_provider_id: The SSO IdP the user used, if any (just used for the prometheus metrics). + should_issue_refresh_token: True if this login should issue + a refresh token alongside the access token. Returns: result: Dictionary of account information after successful login. @@ -274,28 +335,48 @@ class LoginRestServlet(RestServlet): device_id = login_submission.get("device_id") initial_display_name = login_submission.get("initial_device_display_name") - device_id, access_token = await self.registration_handler.register_device( - user_id, device_id, initial_display_name, auth_provider_id=auth_provider_id + ( + device_id, + access_token, + valid_until_ms, + refresh_token, + ) = await self.registration_handler.register_device( + user_id, + device_id, + initial_display_name, + auth_provider_id=auth_provider_id, + should_issue_refresh_token=should_issue_refresh_token, ) - result = { - "user_id": user_id, - "access_token": access_token, - "home_server": self.hs.hostname, - "device_id": device_id, - } + result = LoginResponse( + user_id=user_id, + access_token=access_token, + home_server=self.hs.hostname, + device_id=device_id, + ) + + if valid_until_ms is not None: + expires_in_ms = valid_until_ms - self.clock.time_msec() + result["expires_in_ms"] = expires_in_ms + + if refresh_token is not None: + result["refresh_token"] = refresh_token if callback is not None: await callback(result) return result - async def _do_token_login(self, login_submission: JsonDict) -> Dict[str, str]: + async def _do_token_login( + self, login_submission: JsonDict, should_issue_refresh_token: bool = False + ) -> LoginResponse: """ Handle the final stage of SSO login. Args: - login_submission: The JSON request body. + login_submission: The JSON request body. + should_issue_refresh_token: True if this login should issue + a refresh token alongside the access token. Returns: The body of the JSON response. @@ -309,9 +390,12 @@ class LoginRestServlet(RestServlet): login_submission, self.auth_handler._sso_login_callback, auth_provider_id=res.auth_provider_id, + should_issue_refresh_token=should_issue_refresh_token, ) - async def _do_jwt_login(self, login_submission: JsonDict) -> Dict[str, str]: + async def _do_jwt_login( + self, login_submission: JsonDict, should_issue_refresh_token: bool = False + ) -> LoginResponse: token = login_submission.get("token", None) if token is None: raise LoginError( @@ -342,7 +426,10 @@ class LoginRestServlet(RestServlet): user_id = UserID(user, self.hs.hostname).to_string() result = await self._complete_login( - user_id, login_submission, create_non_existent_users=True + user_id, + login_submission, + create_non_existent_users=True, + should_issue_refresh_token=should_issue_refresh_token, ) return result @@ -371,6 +458,42 @@ def _get_auth_flow_dict_for_idp( return e +class RefreshTokenServlet(RestServlet): + PATTERNS = client_patterns( + "/org.matrix.msc2918.refresh_token/refresh$", releases=(), unstable=True + ) + + def __init__(self, hs: "HomeServer"): + self._auth_handler = hs.get_auth_handler() + self._clock = hs.get_clock() + self.access_token_lifetime = hs.config.access_token_lifetime + + async def on_POST( + self, + request: SynapseRequest, + ): + refresh_submission = parse_json_object_from_request(request) + + assert_params_in_dict(refresh_submission, ["refresh_token"]) + token = refresh_submission["refresh_token"] + if not isinstance(token, str): + raise SynapseError(400, "Invalid param: refresh_token", Codes.INVALID_PARAM) + + valid_until_ms = self._clock.time_msec() + self.access_token_lifetime + access_token, refresh_token = await self._auth_handler.refresh_token( + token, valid_until_ms + ) + expires_in_ms = valid_until_ms - self._clock.time_msec() + return ( + 200, + { + "access_token": access_token, + "refresh_token": refresh_token, + "expires_in_ms": expires_in_ms, + }, + ) + + class SsoRedirectServlet(RestServlet): PATTERNS = list(client_patterns("/login/(cas|sso)/redirect$", v1=True)) + [ re.compile( @@ -477,6 +600,8 @@ class CasTicketServlet(RestServlet): def register_servlets(hs, http_server): LoginRestServlet(hs).register(http_server) + if hs.config.access_token_lifetime is not None: + RefreshTokenServlet(hs).register(http_server) SsoRedirectServlet(hs).register(http_server) if hs.config.cas_enabled: CasTicketServlet(hs).register(http_server) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index a30a5df1b1..4d31584acd 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -41,11 +41,13 @@ from synapse.http.server import finish_request, respond_with_html from synapse.http.servlet import ( RestServlet, assert_params_in_dict, + parse_boolean, parse_json_object_from_request, parse_string, ) from synapse.metrics import threepid_send_requests from synapse.push.mailer import Mailer +from synapse.types import JsonDict from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.stringutils import assert_valid_client_secret, random_string @@ -399,6 +401,7 @@ class RegisterRestServlet(RestServlet): self.password_policy_handler = hs.get_password_policy_handler() self.clock = hs.get_clock() self._registration_enabled = self.hs.config.enable_registration + self._msc2918_enabled = hs.config.access_token_lifetime is not None self._registration_flows = _calculate_registration_flows( hs.config, self.auth_handler @@ -424,6 +427,15 @@ class RegisterRestServlet(RestServlet): "Do not understand membership kind: %s" % (kind.decode("utf8"),) ) + if self._msc2918_enabled: + # Check if this registration should also issue a refresh token, as + # per MSC2918 + should_issue_refresh_token = parse_boolean( + request, name="org.matrix.msc2918.refresh_token", default=False + ) + else: + should_issue_refresh_token = False + # Pull out the provided username and do basic sanity checks early since # the auth layer will store these in sessions. desired_username = None @@ -462,7 +474,10 @@ class RegisterRestServlet(RestServlet): raise SynapseError(400, "Desired Username is missing or not a string") result = await self._do_appservice_registration( - desired_username, access_token, body + desired_username, + access_token, + body, + should_issue_refresh_token=should_issue_refresh_token, ) return 200, result @@ -665,7 +680,9 @@ class RegisterRestServlet(RestServlet): registered = True return_dict = await self._create_registration_details( - registered_user_id, params + registered_user_id, + params, + should_issue_refresh_token=should_issue_refresh_token, ) if registered: @@ -677,7 +694,9 @@ class RegisterRestServlet(RestServlet): return 200, return_dict - async def _do_appservice_registration(self, username, as_token, body): + async def _do_appservice_registration( + self, username, as_token, body, should_issue_refresh_token: bool = False + ): user_id = await self.registration_handler.appservice_register( username, as_token ) @@ -685,19 +704,27 @@ class RegisterRestServlet(RestServlet): user_id, body, is_appservice_ghost=True, + should_issue_refresh_token=should_issue_refresh_token, ) async def _create_registration_details( - self, user_id, params, is_appservice_ghost=False + self, + user_id: str, + params: JsonDict, + is_appservice_ghost: bool = False, + should_issue_refresh_token: bool = False, ): """Complete registration of newly-registered user Allocates device_id if one was not given; also creates access_token. Args: - (str) user_id: full canonical @user:id - (object) params: registration parameters, from which we pull - device_id, initial_device_name and inhibit_login + user_id: full canonical @user:id + params: registration parameters, from which we pull device_id, + initial_device_name and inhibit_login + is_appservice_ghost + should_issue_refresh_token: True if this registration should issue + a refresh token alongside the access token. Returns: dictionary for response from /register """ @@ -705,15 +732,29 @@ class RegisterRestServlet(RestServlet): if not params.get("inhibit_login", False): device_id = params.get("device_id") initial_display_name = params.get("initial_device_display_name") - device_id, access_token = await self.registration_handler.register_device( + ( + device_id, + access_token, + valid_until_ms, + refresh_token, + ) = await self.registration_handler.register_device( user_id, device_id, initial_display_name, is_guest=False, is_appservice_ghost=is_appservice_ghost, + should_issue_refresh_token=should_issue_refresh_token, ) result.update({"access_token": access_token, "device_id": device_id}) + + if valid_until_ms is not None: + expires_in_ms = valid_until_ms - self.clock.time_msec() + result["expires_in_ms"] = expires_in_ms + + if refresh_token is not None: + result["refresh_token"] = refresh_token + return result async def _do_guest_registration(self, params, address=None): @@ -727,19 +768,30 @@ class RegisterRestServlet(RestServlet): # we have nowhere to store it. device_id = synapse.api.auth.GUEST_DEVICE_ID initial_display_name = params.get("initial_device_display_name") - device_id, access_token = await self.registration_handler.register_device( + ( + device_id, + access_token, + valid_until_ms, + refresh_token, + ) = await self.registration_handler.register_device( user_id, device_id, initial_display_name, is_guest=True ) - return ( - 200, - { - "user_id": user_id, - "device_id": device_id, - "access_token": access_token, - "home_server": self.hs.hostname, - }, - ) + result = { + "user_id": user_id, + "device_id": device_id, + "access_token": access_token, + "home_server": self.hs.hostname, + } + + if valid_until_ms is not None: + expires_in_ms = valid_until_ms - self.clock.time_msec() + result["expires_in_ms"] = expires_in_ms + + if refresh_token is not None: + result["refresh_token"] = refresh_token + + return 200, result def _calculate_registration_flows( diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 042e1788b6..ecbbcf3851 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -13,6 +13,7 @@ # limitations under the License. import itertools import logging +from collections import defaultdict from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple from synapse.api.constants import Membership, PresenceState @@ -232,29 +233,51 @@ class SyncRestServlet(RestServlet): ) logger.debug("building sync response dict") - return { - "account_data": {"events": sync_result.account_data}, - "to_device": {"events": sync_result.to_device}, - "device_lists": { - "changed": list(sync_result.device_lists.changed), - "left": list(sync_result.device_lists.left), - }, - "presence": SyncRestServlet.encode_presence(sync_result.presence, time_now), - "rooms": { - Membership.JOIN: joined, - Membership.INVITE: invited, - Membership.KNOCK: knocked, - Membership.LEAVE: archived, - }, - "groups": { - Membership.JOIN: sync_result.groups.join, - Membership.INVITE: sync_result.groups.invite, - Membership.LEAVE: sync_result.groups.leave, - }, - "device_one_time_keys_count": sync_result.device_one_time_keys_count, - "org.matrix.msc2732.device_unused_fallback_key_types": sync_result.device_unused_fallback_key_types, - "next_batch": await sync_result.next_batch.to_string(self.store), - } + + response: dict = defaultdict(dict) + response["next_batch"] = await sync_result.next_batch.to_string(self.store) + + if sync_result.account_data: + response["account_data"] = {"events": sync_result.account_data} + if sync_result.presence: + response["presence"] = SyncRestServlet.encode_presence( + sync_result.presence, time_now + ) + + if sync_result.to_device: + response["to_device"] = {"events": sync_result.to_device} + + if sync_result.device_lists.changed: + response["device_lists"]["changed"] = list(sync_result.device_lists.changed) + if sync_result.device_lists.left: + response["device_lists"]["left"] = list(sync_result.device_lists.left) + + if sync_result.device_one_time_keys_count: + response[ + "device_one_time_keys_count" + ] = sync_result.device_one_time_keys_count + if sync_result.device_unused_fallback_key_types: + response[ + "org.matrix.msc2732.device_unused_fallback_key_types" + ] = sync_result.device_unused_fallback_key_types + + if joined: + response["rooms"][Membership.JOIN] = joined + if invited: + response["rooms"][Membership.INVITE] = invited + if knocked: + response["rooms"][Membership.KNOCK] = knocked + if archived: + response["rooms"][Membership.LEAVE] = archived + + if sync_result.groups.join: + response["groups"][Membership.JOIN] = sync_result.groups.join + if sync_result.groups.invite: + response["groups"][Membership.INVITE] = sync_result.groups.invite + if sync_result.groups.leave: + response["groups"][Membership.LEAVE] = sync_result.groups.leave + + return response @staticmethod def encode_presence(events, time_now): diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index f23f8c6ecf..f2d27ee893 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1075,16 +1075,62 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBas self, origin: str, event_id: str, - ) -> None: - """Remove the given event from the staging area""" - await self.db_pool.simple_delete( - table="federation_inbound_events_staging", - keyvalues={ - "origin": origin, - "event_id": event_id, - }, - desc="remove_received_event_from_staging", - ) + ) -> Optional[int]: + """Remove the given event from the staging area. + + Returns: + The received_ts of the row that was deleted, if any. + """ + if self.db_pool.engine.supports_returning: + + def _remove_received_event_from_staging_txn(txn): + sql = """ + DELETE FROM federation_inbound_events_staging + WHERE origin = ? AND event_id = ? + RETURNING received_ts + """ + + txn.execute(sql, (origin, event_id)) + return txn.fetchone() + + row = await self.db_pool.runInteraction( + "remove_received_event_from_staging", + _remove_received_event_from_staging_txn, + db_autocommit=True, + ) + if row is None: + return None + + return row[0] + + else: + + def _remove_received_event_from_staging_txn(txn): + received_ts = self.db_pool.simple_select_one_onecol_txn( + txn, + table="federation_inbound_events_staging", + keyvalues={ + "origin": origin, + "event_id": event_id, + }, + retcol="received_ts", + allow_none=True, + ) + self.db_pool.simple_delete_txn( + txn, + table="federation_inbound_events_staging", + keyvalues={ + "origin": origin, + "event_id": event_id, + }, + ) + + return received_ts + + return await self.db_pool.runInteraction( + "remove_received_event_from_staging", + _remove_received_event_from_staging_txn, + ) async def get_next_staged_event_id_for_room( self, diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index cbe4be1437..da3a7df27b 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -29,6 +29,25 @@ from synapse.types import JsonDict logger = logging.getLogger(__name__) +_REPLACE_STREAM_ORDRING_SQL_COMMANDS = ( + # there should be no leftover rows without a stream_ordering2, but just in case... + "UPDATE events SET stream_ordering2 = stream_ordering WHERE stream_ordering2 IS NULL", + # finally, we can drop the rule and switch the columns + "DROP RULE populate_stream_ordering2 ON events", + "ALTER TABLE events DROP COLUMN stream_ordering", + "ALTER TABLE events RENAME COLUMN stream_ordering2 TO stream_ordering", +) + + +class _BackgroundUpdates: + EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts" + EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url" + DELETE_SOFT_FAILED_EXTREMITIES = "delete_soft_failed_extremities" + POPULATE_STREAM_ORDERING2 = "populate_stream_ordering2" + INDEX_STREAM_ORDERING2 = "index_stream_ordering2" + REPLACE_STREAM_ORDERING_COLUMN = "replace_stream_ordering_column" + + @attr.s(slots=True, frozen=True) class _CalculateChainCover: """Return value for _calculate_chain_cover_txn.""" @@ -48,19 +67,15 @@ class _CalculateChainCover: class EventsBackgroundUpdatesStore(SQLBaseStore): - - EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts" - EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url" - DELETE_SOFT_FAILED_EXTREMITIES = "delete_soft_failed_extremities" - def __init__(self, database: DatabasePool, db_conn, hs): super().__init__(database, db_conn, hs) self.db_pool.updates.register_background_update_handler( - self.EVENT_ORIGIN_SERVER_TS_NAME, self._background_reindex_origin_server_ts + _BackgroundUpdates.EVENT_ORIGIN_SERVER_TS_NAME, + self._background_reindex_origin_server_ts, ) self.db_pool.updates.register_background_update_handler( - self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, + _BackgroundUpdates.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, self._background_reindex_fields_sender, ) @@ -85,7 +100,8 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): ) self.db_pool.updates.register_background_update_handler( - self.DELETE_SOFT_FAILED_EXTREMITIES, self._cleanup_extremities_bg_update + _BackgroundUpdates.DELETE_SOFT_FAILED_EXTREMITIES, + self._cleanup_extremities_bg_update, ) self.db_pool.updates.register_background_update_handler( @@ -139,6 +155,24 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): self._purged_chain_cover_index, ) + # bg updates for replacing stream_ordering with a BIGINT + # (these only run on postgres.) + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.POPULATE_STREAM_ORDERING2, + self._background_populate_stream_ordering2, + ) + self.db_pool.updates.register_background_index_update( + _BackgroundUpdates.INDEX_STREAM_ORDERING2, + index_name="events_stream_ordering", + table="events", + columns=["stream_ordering2"], + unique=True, + ) + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.REPLACE_STREAM_ORDERING_COLUMN, + self._background_replace_stream_ordering_column, + ) + async def _background_reindex_fields_sender(self, progress, batch_size): target_min_stream_id = progress["target_min_stream_id_inclusive"] max_stream_id = progress["max_stream_id_exclusive"] @@ -190,18 +224,18 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): } self.db_pool.updates._background_update_progress_txn( - txn, self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, progress + txn, _BackgroundUpdates.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, progress ) return len(rows) result = await self.db_pool.runInteraction( - self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, reindex_txn + _BackgroundUpdates.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, reindex_txn ) if not result: await self.db_pool.updates._end_background_update( - self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME + _BackgroundUpdates.EVENT_FIELDS_SENDER_URL_UPDATE_NAME ) return result @@ -264,18 +298,18 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): } self.db_pool.updates._background_update_progress_txn( - txn, self.EVENT_ORIGIN_SERVER_TS_NAME, progress + txn, _BackgroundUpdates.EVENT_ORIGIN_SERVER_TS_NAME, progress ) return len(rows_to_update) result = await self.db_pool.runInteraction( - self.EVENT_ORIGIN_SERVER_TS_NAME, reindex_search_txn + _BackgroundUpdates.EVENT_ORIGIN_SERVER_TS_NAME, reindex_search_txn ) if not result: await self.db_pool.updates._end_background_update( - self.EVENT_ORIGIN_SERVER_TS_NAME + _BackgroundUpdates.EVENT_ORIGIN_SERVER_TS_NAME ) return result @@ -454,7 +488,7 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): if not num_handled: await self.db_pool.updates._end_background_update( - self.DELETE_SOFT_FAILED_EXTREMITIES + _BackgroundUpdates.DELETE_SOFT_FAILED_EXTREMITIES ) def _drop_table_txn(txn): @@ -1009,3 +1043,71 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): await self.db_pool.updates._end_background_update("purged_chain_cover") return result + + async def _background_populate_stream_ordering2( + self, progress: JsonDict, batch_size: int + ) -> int: + """Populate events.stream_ordering2, then replace stream_ordering + + This is to deal with the fact that stream_ordering was initially created as a + 32-bit integer field. + """ + batch_size = max(batch_size, 1) + + def process(txn: Cursor) -> int: + last_stream = progress.get("last_stream", -(1 << 31)) + txn.execute( + """ + UPDATE events SET stream_ordering2=stream_ordering + WHERE stream_ordering IN ( + SELECT stream_ordering FROM events WHERE stream_ordering > ? + ORDER BY stream_ordering LIMIT ? + ) + RETURNING stream_ordering; + """, + (last_stream, batch_size), + ) + row_count = txn.rowcount + if row_count == 0: + return 0 + last_stream = max(row[0] for row in txn) + logger.info("populated stream_ordering2 up to %i", last_stream) + + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.POPULATE_STREAM_ORDERING2, + {"last_stream": last_stream}, + ) + return row_count + + result = await self.db_pool.runInteraction( + "_background_populate_stream_ordering2", process + ) + + if result != 0: + return result + + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.POPULATE_STREAM_ORDERING2 + ) + return 0 + + async def _background_replace_stream_ordering_column( + self, progress: JsonDict, batch_size: int + ) -> int: + """Drop the old 'stream_ordering' column and rename 'stream_ordering2' into its place.""" + + def process(txn: Cursor) -> None: + for sql in _REPLACE_STREAM_ORDRING_SQL_COMMANDS: + logger.info("completing stream_ordering migration: %s", sql) + txn.execute(sql) + + await self.db_pool.runInteraction( + "_background_replace_stream_ordering_column", process + ) + + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.REPLACE_STREAM_ORDERING_COLUMN + ) + + return 0 diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index e5c5cf8ff0..e31c5864ac 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -53,6 +53,9 @@ class TokenLookupResult: valid_until_ms: The timestamp the token expires, if any. token_owner: The "owner" of the token. This is either the same as the user, or a server admin who is logged in as the user. + token_used: True if this token was used at least once in a request. + This field can be out of date since `get_user_by_access_token` is + cached. """ user_id = attr.ib(type=str) @@ -62,6 +65,7 @@ class TokenLookupResult: device_id = attr.ib(type=Optional[str], default=None) valid_until_ms = attr.ib(type=Optional[int], default=None) token_owner = attr.ib(type=str) + token_used = attr.ib(type=bool, default=False) # Make the token owner default to the user ID, which is the common case. @token_owner.default @@ -69,6 +73,29 @@ class TokenLookupResult: return self.user_id +@attr.s(frozen=True, slots=True) +class RefreshTokenLookupResult: + """Result of looking up a refresh token.""" + + user_id = attr.ib(type=str) + """The user this token belongs to.""" + + device_id = attr.ib(type=str) + """The device associated with this refresh token.""" + + token_id = attr.ib(type=int) + """The ID of this refresh token.""" + + next_token_id = attr.ib(type=Optional[int]) + """The ID of the refresh token which replaced this one.""" + + has_next_refresh_token_been_refreshed = attr.ib(type=bool) + """True if the next refresh token was used for another refresh.""" + + has_next_access_token_been_used = attr.ib(type=bool) + """True if the next access token was already used at least once.""" + + class RegistrationWorkerStore(CacheInvalidationWorkerStore): def __init__( self, @@ -441,7 +468,8 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): access_tokens.id as token_id, access_tokens.device_id, access_tokens.valid_until_ms, - access_tokens.user_id as token_owner + access_tokens.user_id as token_owner, + access_tokens.used as token_used FROM users INNER JOIN access_tokens on users.name = COALESCE(puppets_user_id, access_tokens.user_id) WHERE token = ? @@ -449,8 +477,15 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): txn.execute(sql, (token,)) rows = self.db_pool.cursor_to_dict(txn) + if rows: - return TokenLookupResult(**rows[0]) + row = rows[0] + + # This field is nullable, ensure it comes out as a boolean + if row["token_used"] is None: + row["token_used"] = False + + return TokenLookupResult(**row) return None @@ -1072,6 +1107,111 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): desc="update_access_token_last_validated", ) + @cached() + async def mark_access_token_as_used(self, token_id: int) -> None: + """ + Mark the access token as used, which invalidates the refresh token used + to obtain it. + + Because get_user_by_access_token is cached, this function might be + called multiple times for the same token, effectively doing unnecessary + SQL updates. Because updating the `used` field only goes one way (from + False to True) it is safe to cache this function as well to avoid this + issue. + + Args: + token_id: The ID of the access token to update. + Raises: + StoreError if there was a problem updating this. + """ + await self.db_pool.simple_update_one( + "access_tokens", + {"id": token_id}, + {"used": True}, + desc="mark_access_token_as_used", + ) + + async def lookup_refresh_token( + self, token: str + ) -> Optional[RefreshTokenLookupResult]: + """Lookup a refresh token with hints about its validity.""" + + def _lookup_refresh_token_txn(txn) -> Optional[RefreshTokenLookupResult]: + txn.execute( + """ + SELECT + rt.id token_id, + rt.user_id, + rt.device_id, + rt.next_token_id, + (nrt.next_token_id IS NOT NULL) has_next_refresh_token_been_refreshed, + at.used has_next_access_token_been_used + FROM refresh_tokens rt + LEFT JOIN refresh_tokens nrt ON rt.next_token_id = nrt.id + LEFT JOIN access_tokens at ON at.refresh_token_id = nrt.id + WHERE rt.token = ? + """, + (token,), + ) + row = txn.fetchone() + + if row is None: + return None + + return RefreshTokenLookupResult( + token_id=row[0], + user_id=row[1], + device_id=row[2], + next_token_id=row[3], + has_next_refresh_token_been_refreshed=row[4], + # This column is nullable, ensure it's a boolean + has_next_access_token_been_used=(row[5] or False), + ) + + return await self.db_pool.runInteraction( + "lookup_refresh_token", _lookup_refresh_token_txn + ) + + async def replace_refresh_token(self, token_id: int, next_token_id: int) -> None: + """ + Set the successor of a refresh token, removing the existing successor + if any. + + Args: + token_id: ID of the refresh token to update. + next_token_id: ID of its successor. + """ + + def _replace_refresh_token_txn(txn) -> None: + # First check if there was an existing refresh token + old_next_token_id = self.db_pool.simple_select_one_onecol_txn( + txn, + "refresh_tokens", + {"id": token_id}, + "next_token_id", + allow_none=True, + ) + + self.db_pool.simple_update_one_txn( + txn, + "refresh_tokens", + {"id": token_id}, + {"next_token_id": next_token_id}, + ) + + # Delete the old "next" token if it exists. This should cascade and + # delete the associated access_token + if old_next_token_id is not None: + self.db_pool.simple_delete_one_txn( + txn, + "refresh_tokens", + {"id": old_next_token_id}, + ) + + await self.db_pool.runInteraction( + "replace_refresh_token", _replace_refresh_token_txn + ) + class RegistrationBackgroundUpdateStore(RegistrationWorkerStore): def __init__( @@ -1263,6 +1403,7 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): self._ignore_unknown_session_error = hs.config.request_token_inhibit_3pid_errors self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id") + self._refresh_tokens_id_gen = IdGenerator(db_conn, "refresh_tokens", "id") async def add_access_token_to_user( self, @@ -1271,14 +1412,18 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): device_id: Optional[str], valid_until_ms: Optional[int], puppets_user_id: Optional[str] = None, + refresh_token_id: Optional[int] = None, ) -> int: """Adds an access token for the given user. Args: user_id: The user ID. token: The new access token to add. - device_id: ID of the device to associate with the access token + device_id: ID of the device to associate with the access token. valid_until_ms: when the token is valid until. None for no expiry. + puppets_user_id + refresh_token_id: ID of the refresh token generated alongside this + access token. Raises: StoreError if there was a problem adding this. Returns: @@ -1297,12 +1442,47 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): "valid_until_ms": valid_until_ms, "puppets_user_id": puppets_user_id, "last_validated": now, + "refresh_token_id": refresh_token_id, + "used": False, }, desc="add_access_token_to_user", ) return next_id + async def add_refresh_token_to_user( + self, + user_id: str, + token: str, + device_id: Optional[str], + ) -> int: + """Adds a refresh token for the given user. + + Args: + user_id: The user ID. + token: The new access token to add. + device_id: ID of the device to associate with the refresh token. + Raises: + StoreError if there was a problem adding this. + Returns: + The token ID + """ + next_id = self._refresh_tokens_id_gen.get_next() + + await self.db_pool.simple_insert( + "refresh_tokens", + { + "id": next_id, + "user_id": user_id, + "device_id": device_id, + "token": token, + "next_token_id": None, + }, + desc="add_refresh_token_to_user", + ) + + return next_id + def _set_device_for_access_token_txn(self, txn, token: str, device_id: str) -> str: old_device_id = self.db_pool.simple_select_one_onecol_txn( txn, "access_tokens", {"token": token}, "device_id" @@ -1545,7 +1725,7 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): device_id: Optional[str] = None, ) -> List[Tuple[str, int, Optional[str]]]: """ - Invalidate access tokens belonging to a user + Invalidate access and refresh tokens belonging to a user Args: user_id: ID of user the tokens belong to @@ -1565,7 +1745,13 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): items = keyvalues.items() where_clause = " AND ".join(k + " = ?" for k, _ in items) values = [v for _, v in items] # type: List[Union[str, int]] + # Conveniently, refresh_tokens and access_tokens both use the user_id and device_id fields. Only caveat + # is the `except_token_id` param that is tricky to get right, so for now we're just using the same where + # clause and values before we handle that. This seems to be only used in the "set password" handler. + refresh_where_clause = where_clause + refresh_values = values.copy() if except_token_id: + # TODO: support that for refresh tokens where_clause += " AND id != ?" values.append(except_token_id) @@ -1583,6 +1769,11 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): txn.execute("DELETE FROM access_tokens WHERE %s" % where_clause, values) + txn.execute( + "DELETE FROM refresh_tokens WHERE %s" % refresh_where_clause, + refresh_values, + ) + return tokens_and_devices return await self.db_pool.runInteraction("user_delete_access_tokens", f) @@ -1599,6 +1790,14 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): await self.db_pool.runInteraction("delete_access_token", f) + async def delete_refresh_token(self, refresh_token: str) -> None: + def f(txn): + self.db_pool.simple_delete_one_txn( + txn, table="refresh_tokens", keyvalues={"token": refresh_token} + ) + + await self.db_pool.runInteraction("delete_refresh_token", f) + async def add_user_pending_deactivation(self, user_id: str) -> None: """ Adds a user to the table of users who need to be parted from all the rooms they're diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py index 1882bfd9cf..20cd63c330 100644 --- a/synapse/storage/engines/_base.py +++ b/synapse/storage/engines/_base.py @@ -49,6 +49,12 @@ class BaseDatabaseEngine(Generic[ConnectionType], metaclass=abc.ABCMeta): """ ... + @property + @abc.abstractmethod + def supports_returning(self) -> bool: + """Do we support the `RETURNING` clause in insert/update/delete?""" + ... + @abc.abstractmethod def check_database( self, db_conn: ConnectionType, allow_outdated_version: bool = False diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 21411c5fea..30f948a0f7 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -133,6 +133,11 @@ class PostgresEngine(BaseDatabaseEngine): """Do we support using `a = ANY(?)` and passing a list""" return True + @property + def supports_returning(self) -> bool: + """Do we support the `RETURNING` clause in insert/update/delete?""" + return True + def is_deadlock(self, error): if isinstance(error, self.module.DatabaseError): # https://www.postgresql.org/docs/current/static/errcodes-appendix.html diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 5fe1b205e1..70d17d4f2c 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -60,6 +60,11 @@ class Sqlite3Engine(BaseDatabaseEngine["sqlite3.Connection"]): """Do we support using `a = ANY(?)` and passing a list""" return False + @property + def supports_returning(self) -> bool: + """Do we support the `RETURNING` clause in insert/update/delete?""" + return self.module.sqlite_version_info >= (3, 35, 0) + def check_database(self, db_conn, allow_outdated_version: bool = False): if not allow_outdated_version: version = self.module.sqlite_version_info diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index d36ba1d773..0a53b73ccc 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SCHEMA_VERSION = 59 +SCHEMA_VERSION = 60 """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the diff --git a/synapse/storage/schema/main/delta/59/14refresh_tokens.sql b/synapse/storage/schema/main/delta/59/14refresh_tokens.sql new file mode 100644 index 0000000000..9a6bce1e3e --- /dev/null +++ b/synapse/storage/schema/main/delta/59/14refresh_tokens.sql @@ -0,0 +1,34 @@ +/* Copyright 2021 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Holds MSC2918 refresh tokens +CREATE TABLE refresh_tokens ( + id BIGINT PRIMARY KEY, + user_id TEXT NOT NULL, + device_id TEXT NOT NULL, + token TEXT NOT NULL, + -- When consumed, a new refresh token is generated, which is tracked by + -- this foreign key + next_token_id BIGINT REFERENCES refresh_tokens (id) ON DELETE CASCADE, + UNIQUE(token) +); + +-- Add a reference to the refresh token generated alongside each access token +ALTER TABLE "access_tokens" + ADD COLUMN refresh_token_id BIGINT REFERENCES refresh_tokens (id) ON DELETE CASCADE; + +-- Add a flag whether the token was already used or not +ALTER TABLE "access_tokens" + ADD COLUMN used BOOLEAN; diff --git a/synapse/storage/schema/main/delta/60/01recreate_stream_ordering.sql.postgres b/synapse/storage/schema/main/delta/60/01recreate_stream_ordering.sql.postgres new file mode 100644 index 0000000000..88c9f8bd0d --- /dev/null +++ b/synapse/storage/schema/main/delta/60/01recreate_stream_ordering.sql.postgres @@ -0,0 +1,40 @@ +/* Copyright 2021 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- This migration handles the process of changing the type of `stream_ordering` to +-- a BIGINT. +-- +-- Note that this is only a problem on postgres as sqlite only has one "integer" type +-- which can cope with values up to 2^63. + +-- First add a new column to contain the bigger stream_ordering +ALTER TABLE events ADD COLUMN stream_ordering2 BIGINT; + +-- Create a rule which will populate it for new rows. +CREATE OR REPLACE RULE "populate_stream_ordering2" AS + ON INSERT TO events + DO UPDATE events SET stream_ordering2=NEW.stream_ordering WHERE stream_ordering=NEW.stream_ordering; + +-- Start a bg process to populate it for old events +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (6001, 'populate_stream_ordering2', '{}'); + +-- ... and another to build an index on it +INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES + (6001, 'index_stream_ordering2', '{}', 'populate_stream_ordering2'); + +-- ... and another to do the switcheroo +INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES + (6001, 'replace_stream_ordering_column', '{}', 'index_stream_ordering2'); diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 1b0a815757..f76fea4f66 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -58,6 +58,7 @@ class AuthTestCase(unittest.HomeserverTestCase): user_id=self.test_user, token_id=5, device_id="device" ) self.store.get_user_by_access_token = simple_async_mock(user_info) + self.store.mark_access_token_as_used = simple_async_mock(None) request = Mock(args={}) request.args[b"access_token"] = [self.test_token] diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 8c215d50f2..aab44bce4a 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -205,9 +205,7 @@ class FederationKnockingTestCase( # Have this homeserver skip event auth checks. This is necessary due to # event auth checks ensuring that events were signed by the sender's homeserver. - async def _check_event_auth( - origin, event, context, state, auth_events, backfilled - ): + async def _check_event_auth(origin, event, context, *args, **kwargs): return context homeserver.get_federation_handler()._check_event_auth = _check_event_auth diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 84c38b295d..3ac48e5e95 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -257,7 +257,7 @@ class DehydrationTestCase(unittest.HomeserverTestCase): self.assertEqual(device_data, {"device_data": {"foo": "bar"}}) # Create a new login for the user and dehydrated the device - device_id, access_token = self.get_success( + device_id, access_token, _expiration_time, _refresh_token = self.get_success( self.registration.register_device( user_id=user_id, device_id=None, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 8796af45ed..ba8cf44f46 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -251,7 +251,7 @@ class FederationTestCase(unittest.HomeserverTestCase): join_event.signatures[other_server] = {"x": "y"} with LoggingContext("send_join"): d = run_in_background( - self.handler.on_send_join_request, other_server, join_event + self.handler.on_send_membership_event, other_server, join_event ) self.get_success(d) diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index c5f6bc3c75..d3efb67e3e 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -19,7 +19,7 @@ from synapse.api.constants import UserTypes from synapse.api.errors import Codes, ResourceLimitError, SynapseError from synapse.events.spamcheck import load_legacy_spam_checkers from synapse.spam_checker_api import RegistrationBehaviour -from synapse.types import RoomAlias, UserID, create_requester +from synapse.types import RoomAlias, RoomID, UserID, create_requester from tests.test_utils import make_awaitable from tests.unittest import override_config @@ -719,3 +719,50 @@ class RegistrationTestCase(unittest.HomeserverTestCase): ) return user_id, token + + +class RemoteAutoJoinTestCase(unittest.HomeserverTestCase): + """Tests auto-join on remote rooms.""" + + def make_homeserver(self, reactor, clock): + self.room_id = "!roomid:remotetest" + + async def update_membership(*args, **kwargs): + pass + + async def lookup_room_alias(*args, **kwargs): + return RoomID.from_string(self.room_id), ["remotetest"] + + self.room_member_handler = Mock(spec=["update_membership", "lookup_room_alias"]) + self.room_member_handler.update_membership.side_effect = update_membership + self.room_member_handler.lookup_room_alias.side_effect = lookup_room_alias + + hs = self.setup_test_homeserver(room_member_handler=self.room_member_handler) + return hs + + def prepare(self, reactor, clock, hs): + self.handler = self.hs.get_registration_handler() + self.store = self.hs.get_datastore() + + @override_config({"auto_join_rooms": ["#room:remotetest"]}) + def test_auto_create_auto_join_remote_room(self): + """Tests that we don't attempt to create remote rooms, and that we don't attempt + to invite ourselves to rooms we're not in.""" + + # Register a first user; this should call _create_and_join_rooms + self.get_success(self.handler.register_user(localpart="jeff")) + + _, kwargs = self.room_member_handler.update_membership.call_args + + self.assertEqual(kwargs["room_id"], self.room_id) + self.assertEqual(kwargs["action"], "join") + self.assertEqual(kwargs["remote_room_hosts"], ["remotetest"]) + + # Register a second user; this should call _join_rooms + self.get_success(self.handler.register_user(localpart="jeff2")) + + _, kwargs = self.room_member_handler.update_membership.call_args + + self.assertEqual(kwargs["room_id"], self.room_id) + self.assertEqual(kwargs["action"], "join") + self.assertEqual(kwargs["remote_room_hosts"], ["remotetest"]) diff --git a/tests/handlers/test_space_summary.py b/tests/handlers/test_space_summary.py index 131d362ccc..9771d3fb3b 100644 --- a/tests/handlers/test_space_summary.py +++ b/tests/handlers/test_space_summary.py @@ -14,6 +14,7 @@ from typing import Any, Iterable, Optional, Tuple from unittest import mock +from synapse.api.constants import EventContentFields, RoomTypes from synapse.api.errors import AuthError from synapse.handlers.space_summary import _child_events_comparison_key from synapse.rest import admin @@ -97,9 +98,21 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): self.hs = hs self.handler = self.hs.get_space_summary_handler() + # Create a user. self.user = self.register_user("user", "pass") self.token = self.login("user", "pass") + # Create a space and a child room. + self.space = self.helper.create_room_as( + self.user, + tok=self.token, + extra_content={ + "creation_content": {EventContentFields.ROOM_TYPE: RoomTypes.SPACE} + }, + ) + self.room = self.helper.create_room_as(self.user, tok=self.token) + self._add_child(self.space, self.room, self.token) + def _add_child(self, space_id: str, room_id: str, token: str) -> None: """Add a child room to a space.""" self.helper.send_state( @@ -128,43 +141,32 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): def test_simple_space(self): """Test a simple space with a single room.""" - space = self.helper.create_room_as(self.user, tok=self.token) - room = self.helper.create_room_as(self.user, tok=self.token) - self._add_child(space, room, self.token) - - result = self.get_success(self.handler.get_space_summary(self.user, space)) + result = self.get_success(self.handler.get_space_summary(self.user, self.space)) # The result should have the space and the room in it, along with a link # from space -> room. - self._assert_rooms(result, [space, room]) - self._assert_events(result, [(space, room)]) + self._assert_rooms(result, [self.space, self.room]) + self._assert_events(result, [(self.space, self.room)]) def test_visibility(self): """A user not in a space cannot inspect it.""" - space = self.helper.create_room_as(self.user, tok=self.token) - room = self.helper.create_room_as(self.user, tok=self.token) - self._add_child(space, room, self.token) - user2 = self.register_user("user2", "pass") token2 = self.login("user2", "pass") # The user cannot see the space. - self.get_failure(self.handler.get_space_summary(user2, space), AuthError) + self.get_failure(self.handler.get_space_summary(user2, self.space), AuthError) # Joining the room causes it to be visible. - self.helper.join(space, user2, tok=token2) - result = self.get_success(self.handler.get_space_summary(user2, space)) + self.helper.join(self.space, user2, tok=token2) + result = self.get_success(self.handler.get_space_summary(user2, self.space)) # The result should only have the space, but includes the link to the room. - self._assert_rooms(result, [space]) - self._assert_events(result, [(space, room)]) + self._assert_rooms(result, [self.space]) + self._assert_events(result, [(self.space, self.room)]) def test_world_readable(self): """A world-readable room is visible to everyone.""" - space = self.helper.create_room_as(self.user, tok=self.token) - room = self.helper.create_room_as(self.user, tok=self.token) - self._add_child(space, room, self.token) self.helper.send_state( - space, + self.space, event_type="m.room.history_visibility", body={"history_visibility": "world_readable"}, tok=self.token, @@ -173,6 +175,6 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): user2 = self.register_user("user2", "pass") # The space should be visible, as well as the link to the room. - result = self.get_success(self.handler.get_space_summary(user2, space)) - self._assert_rooms(result, [space]) - self._assert_events(result, [(space, room)]) + result = self.get_success(self.handler.get_space_summary(user2, self.space)) + self._assert_rooms(result, [self.space]) + self._assert_events(result, [(self.space, self.room)]) diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 584da58371..a0c710f855 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -228,7 +228,7 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): builder.build(prev_event_ids=prev_event_ids, auth_event_ids=None) ) - self.get_success(federation.on_send_join_request(remote_server, join_event)) + self.get_success(federation.on_send_membership_event(remote_server, join_event)) self.replicate() return room diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index ed55a640af..69798e95c3 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -52,6 +52,7 @@ class RestHelper: room_version: str = None, tok: str = None, expect_code: int = 200, + extra_content: Optional[Dict] = None, ) -> str: """ Create a room. @@ -72,7 +73,7 @@ class RestHelper: temp_id = self.auth_user_id self.auth_user_id = room_creator path = "/_matrix/client/r0/createRoom" - content = {} + content = extra_content or {} if not is_public: content["visibility"] = "private" if room_version: diff --git a/tests/rest/client/v2_alpha/test_auth.py b/tests/rest/client/v2_alpha/test_auth.py index 485e3650c3..6b90f838b6 100644 --- a/tests/rest/client/v2_alpha/test_auth.py +++ b/tests/rest/client/v2_alpha/test_auth.py @@ -20,7 +20,7 @@ import synapse.rest.admin from synapse.api.constants import LoginType from synapse.handlers.ui_auth.checkers import UserInteractiveAuthChecker from synapse.rest.client.v1 import login -from synapse.rest.client.v2_alpha import auth, devices, register +from synapse.rest.client.v2_alpha import account, auth, devices, register from synapse.rest.synapse.client import build_synapse_client_resource_tree from synapse.types import JsonDict, UserID @@ -498,3 +498,221 @@ class UIAuthTests(unittest.HomeserverTestCase): self.delete_device( self.user_tok, self.device_id, 403, body={"auth": {"session": session_id}} ) + + +class RefreshAuthTests(unittest.HomeserverTestCase): + servlets = [ + auth.register_servlets, + account.register_servlets, + login.register_servlets, + synapse.rest.admin.register_servlets_for_client_rest_resource, + register.register_servlets, + ] + hijack_auth = False + + def prepare(self, reactor, clock, hs): + self.user_pass = "pass" + self.user = self.register_user("test", self.user_pass) + + def test_login_issue_refresh_token(self): + """ + A login response should include a refresh_token only if asked. + """ + # Test login + body = {"type": "m.login.password", "user": "test", "password": self.user_pass} + + login_without_refresh = self.make_request( + "POST", "/_matrix/client/r0/login", body + ) + self.assertEqual(login_without_refresh.code, 200, login_without_refresh.result) + self.assertNotIn("refresh_token", login_without_refresh.json_body) + + login_with_refresh = self.make_request( + "POST", + "/_matrix/client/r0/login?org.matrix.msc2918.refresh_token=true", + body, + ) + self.assertEqual(login_with_refresh.code, 200, login_with_refresh.result) + self.assertIn("refresh_token", login_with_refresh.json_body) + self.assertIn("expires_in_ms", login_with_refresh.json_body) + + def test_register_issue_refresh_token(self): + """ + A register response should include a refresh_token only if asked. + """ + register_without_refresh = self.make_request( + "POST", + "/_matrix/client/r0/register", + { + "username": "test2", + "password": self.user_pass, + "auth": {"type": LoginType.DUMMY}, + }, + ) + self.assertEqual( + register_without_refresh.code, 200, register_without_refresh.result + ) + self.assertNotIn("refresh_token", register_without_refresh.json_body) + + register_with_refresh = self.make_request( + "POST", + "/_matrix/client/r0/register?org.matrix.msc2918.refresh_token=true", + { + "username": "test3", + "password": self.user_pass, + "auth": {"type": LoginType.DUMMY}, + }, + ) + self.assertEqual(register_with_refresh.code, 200, register_with_refresh.result) + self.assertIn("refresh_token", register_with_refresh.json_body) + self.assertIn("expires_in_ms", register_with_refresh.json_body) + + def test_token_refresh(self): + """ + A refresh token can be used to issue a new access token. + """ + body = {"type": "m.login.password", "user": "test", "password": self.user_pass} + login_response = self.make_request( + "POST", + "/_matrix/client/r0/login?org.matrix.msc2918.refresh_token=true", + body, + ) + self.assertEqual(login_response.code, 200, login_response.result) + + refresh_response = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh", + {"refresh_token": login_response.json_body["refresh_token"]}, + ) + self.assertEqual(refresh_response.code, 200, refresh_response.result) + self.assertIn("access_token", refresh_response.json_body) + self.assertIn("refresh_token", refresh_response.json_body) + self.assertIn("expires_in_ms", refresh_response.json_body) + + # The access and refresh tokens should be different from the original ones after refresh + self.assertNotEqual( + login_response.json_body["access_token"], + refresh_response.json_body["access_token"], + ) + self.assertNotEqual( + login_response.json_body["refresh_token"], + refresh_response.json_body["refresh_token"], + ) + + @override_config({"access_token_lifetime": "1m"}) + def test_refresh_token_expiration(self): + """ + The access token should have some time as specified in the config. + """ + body = {"type": "m.login.password", "user": "test", "password": self.user_pass} + login_response = self.make_request( + "POST", + "/_matrix/client/r0/login?org.matrix.msc2918.refresh_token=true", + body, + ) + self.assertEqual(login_response.code, 200, login_response.result) + self.assertApproximates( + login_response.json_body["expires_in_ms"], 60 * 1000, 100 + ) + + refresh_response = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh", + {"refresh_token": login_response.json_body["refresh_token"]}, + ) + self.assertEqual(refresh_response.code, 200, refresh_response.result) + self.assertApproximates( + refresh_response.json_body["expires_in_ms"], 60 * 1000, 100 + ) + + def test_refresh_token_invalidation(self): + """Refresh tokens are invalidated after first use of the next token. + + A refresh token is considered invalid if: + - it was already used at least once + - and either + - the next access token was used + - the next refresh token was used + + The chain of tokens goes like this: + + login -|-> first_refresh -> third_refresh (fails) + |-> second_refresh -> fifth_refresh + |-> fourth_refresh (fails) + """ + + body = {"type": "m.login.password", "user": "test", "password": self.user_pass} + login_response = self.make_request( + "POST", + "/_matrix/client/r0/login?org.matrix.msc2918.refresh_token=true", + body, + ) + self.assertEqual(login_response.code, 200, login_response.result) + + # This first refresh should work properly + first_refresh_response = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh", + {"refresh_token": login_response.json_body["refresh_token"]}, + ) + self.assertEqual( + first_refresh_response.code, 200, first_refresh_response.result + ) + + # This one as well, since the token in the first one was never used + second_refresh_response = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh", + {"refresh_token": login_response.json_body["refresh_token"]}, + ) + self.assertEqual( + second_refresh_response.code, 200, second_refresh_response.result + ) + + # This one should not, since the token from the first refresh is not valid anymore + third_refresh_response = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh", + {"refresh_token": first_refresh_response.json_body["refresh_token"]}, + ) + self.assertEqual( + third_refresh_response.code, 401, third_refresh_response.result + ) + + # The associated access token should also be invalid + whoami_response = self.make_request( + "GET", + "/_matrix/client/r0/account/whoami", + access_token=first_refresh_response.json_body["access_token"], + ) + self.assertEqual(whoami_response.code, 401, whoami_response.result) + + # But all other tokens should work (they will expire after some time) + for access_token in [ + second_refresh_response.json_body["access_token"], + login_response.json_body["access_token"], + ]: + whoami_response = self.make_request( + "GET", "/_matrix/client/r0/account/whoami", access_token=access_token + ) + self.assertEqual(whoami_response.code, 200, whoami_response.result) + + # Now that the access token from the last valid refresh was used once, refreshing with the N-1 token should fail + fourth_refresh_response = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh", + {"refresh_token": login_response.json_body["refresh_token"]}, + ) + self.assertEqual( + fourth_refresh_response.code, 403, fourth_refresh_response.result + ) + + # But refreshing from the last valid refresh token still works + fifth_refresh_response = self.make_request( + "POST", + "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh", + {"refresh_token": second_refresh_response.json_body["refresh_token"]}, + ) + self.assertEqual( + fifth_refresh_response.code, 200, fifth_refresh_response.result + ) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 012910f136..cdca3a3e23 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -41,35 +41,7 @@ class FilterTestCase(unittest.HomeserverTestCase): channel = self.make_request("GET", "/sync") self.assertEqual(channel.code, 200) - self.assertTrue( - { - "next_batch", - "rooms", - "presence", - "account_data", - "to_device", - "device_lists", - }.issubset(set(channel.json_body.keys())) - ) - - def test_sync_presence_disabled(self): - """ - When presence is disabled, the key does not appear in /sync. - """ - self.hs.config.use_presence = False - - channel = self.make_request("GET", "/sync") - - self.assertEqual(channel.code, 200) - self.assertTrue( - { - "next_batch", - "rooms", - "account_data", - "to_device", - "device_lists", - }.issubset(set(channel.json_body.keys())) - ) + self.assertIn("next_batch", channel.json_body) class SyncFilterTestCase(unittest.HomeserverTestCase): diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py index d46521ccdc..3245aa91ca 100644 --- a/tests/server_notices/test_resource_limits_server_notices.py +++ b/tests/server_notices/test_resource_limits_server_notices.py @@ -306,8 +306,9 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase): channel = self.make_request("GET", "/sync?timeout=0", access_token=tok) - invites = channel.json_body["rooms"]["invite"] - self.assertEqual(len(invites), 0, invites) + self.assertNotIn( + "rooms", channel.json_body, "Got invites without server notice" + ) def test_invite_with_notice(self): """Tests that, if the MAU limit is hit, the server notices user invites each user @@ -364,7 +365,8 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase): # We could also pick another user and sync with it, which would return an # invite to a system notices room, but it doesn't matter which user we're # using so we use the last one because it saves us an extra sync. - invites = channel.json_body["rooms"]["invite"] + if "rooms" in channel.json_body: + invites = channel.json_body["rooms"]["invite"] # Make sure we have an invite to process. self.assertEqual(len(invites), 1, invites) |