diff options
Diffstat (limited to 'latest/print.html')
| -rw-r--r-- | latest/print.html | 695 |
1 files changed, 650 insertions, 45 deletions
diff --git a/latest/print.html b/latest/print.html index 28b8044829..3835eb1f74 100644 --- a/latest/print.html +++ b/latest/print.html @@ -77,7 +77,7 @@ <nav id="sidebar" class="sidebar" aria-label="Table of contents"> <div class="sidebar-scrollbox"> - <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="federate.html">Federation</a></li><li class="chapter-item expanded "><a href="usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/configuration/config_documentation.html">Configuration Manual</a></li><li class="chapter-item expanded "><a href="usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="templates.html">Templates</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="jwt.html">JSON Web Tokens</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/refresh_tokens.html">Refresh Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="development/url_previews.html">URL Previews</a></li><li class="chapter-item expanded "><a href="user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="modules/account_data_callbacks.html">Account data callbacks</a></li><li class="chapter-item expanded "><a href="modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="development/releases.html">Release Cycle</a></li><li class="chapter-item expanded "><a href="development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/demo.html">Demo scripts</a></li></ol></li><li class="chapter-item expanded "><a href="opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="replication.html">Replication</a></li><li class="chapter-item expanded "><a href="tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> + <ol class="chapter"><li class="chapter-item expanded affix "><li class="part-title">Introduction</li><li class="chapter-item expanded "><a href="welcome_and_overview.html">Welcome and Overview</a></li><li class="chapter-item expanded affix "><li class="part-title">Setup</li><li class="chapter-item expanded "><a href="setup/installation.html">Installation</a></li><li class="chapter-item expanded "><a href="postgres.html">Using Postgres</a></li><li class="chapter-item expanded "><a href="reverse_proxy.html">Configuring a Reverse Proxy</a></li><li class="chapter-item expanded "><a href="setup/forward_proxy.html">Configuring a Forward/Outbound Proxy</a></li><li class="chapter-item expanded "><a href="turn-howto.html">Configuring a Turn Server</a></li><li class="chapter-item expanded "><a href="delegate.html">Delegation</a></li><li class="chapter-item expanded affix "><li class="part-title">Upgrading</li><li class="chapter-item expanded "><a href="upgrade.html">Upgrading between Synapse Versions</a></li><li class="chapter-item expanded affix "><li class="part-title">Usage</li><li class="chapter-item expanded "><a href="federate.html">Federation</a></li><li class="chapter-item expanded "><a href="usage/configuration/index.html">Configuration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/configuration/config_documentation.html">Configuration Manual</a></li><li class="chapter-item expanded "><a href="usage/configuration/homeserver_sample_config.html">Homeserver Sample Config File</a></li><li class="chapter-item expanded "><a href="usage/configuration/logging_sample_config.html">Logging Sample Config File</a></li><li class="chapter-item expanded "><a href="structured_logging.html">Structured Logging</a></li><li class="chapter-item expanded "><a href="templates.html">Templates</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/index.html">User Authentication</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/index.html">Single-Sign On</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="openid.html">OpenID Connect</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/single_sign_on/cas.html">CAS</a></li><li class="chapter-item expanded "><a href="sso_mapping_providers.html">SSO Mapping Providers</a></li></ol></li><li class="chapter-item expanded "><a href="password_auth_providers.html">Password Auth Providers</a></li><li class="chapter-item expanded "><a href="jwt.html">JSON Web Tokens</a></li><li class="chapter-item expanded "><a href="usage/configuration/user_authentication/refresh_tokens.html">Refresh Tokens</a></li></ol></li><li class="chapter-item expanded "><a href="CAPTCHA_SETUP.html">Registration Captcha</a></li><li class="chapter-item expanded "><a href="application_services.html">Application Services</a></li><li class="chapter-item expanded "><a href="server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="consent_tracking.html">Consent Tracking</a></li><li class="chapter-item expanded "><a href="development/url_previews.html">URL Previews</a></li><li class="chapter-item expanded "><a href="user_directory.html">User Directory</a></li><li class="chapter-item expanded "><a href="message_retention_policies.html">Message Retention Policies</a></li><li class="chapter-item expanded "><a href="modules/index.html">Pluggable Modules</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="modules/writing_a_module.html">Writing a module</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="modules/spam_checker_callbacks.html">Spam checker callbacks</a></li><li class="chapter-item expanded "><a href="modules/third_party_rules_callbacks.html">Third-party rules callbacks</a></li><li class="chapter-item expanded "><a href="modules/presence_router_callbacks.html">Presence router callbacks</a></li><li class="chapter-item expanded "><a href="modules/account_validity_callbacks.html">Account validity callbacks</a></li><li class="chapter-item expanded "><a href="modules/password_auth_provider_callbacks.html">Password auth provider callbacks</a></li><li class="chapter-item expanded "><a href="modules/background_update_controller_callbacks.html">Background update controller callbacks</a></li><li class="chapter-item expanded "><a href="modules/account_data_callbacks.html">Account data callbacks</a></li><li class="chapter-item expanded "><a href="modules/porting_legacy_module.html">Porting a legacy module to the new interface</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="workers.html">Workers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="synctl_workers.html">Using synctl with Workers</a></li><li class="chapter-item expanded "><a href="systemd-with-workers/index.html">Systemd</a></li></ol></li></ol></li><li class="chapter-item expanded "><a href="usage/administration/index.html">Administration</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="usage/administration/admin_api/index.html">Admin API</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="admin_api/account_validity.html">Account Validity</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/background_updates.html">Background Updates</a></li><li class="chapter-item expanded "><a href="admin_api/delete_group.html">Delete Group</a></li><li class="chapter-item expanded "><a href="admin_api/event_reports.html">Event Reports</a></li><li class="chapter-item expanded "><a href="admin_api/media_admin_api.html">Media</a></li><li class="chapter-item expanded "><a href="admin_api/purge_history_api.html">Purge History</a></li><li class="chapter-item expanded "><a href="admin_api/register_api.html">Register Users</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/registration_tokens.html">Registration Tokens</a></li><li class="chapter-item expanded "><a href="admin_api/room_membership.html">Manipulate Room Membership</a></li><li class="chapter-item expanded "><a href="admin_api/rooms.html">Rooms</a></li><li class="chapter-item expanded "><a href="admin_api/server_notices.html">Server Notices</a></li><li class="chapter-item expanded "><a href="admin_api/statistics.html">Statistics</a></li><li class="chapter-item expanded "><a href="admin_api/user_admin_api.html">Users</a></li><li class="chapter-item expanded "><a href="admin_api/version_api.html">Server Version</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_api/federation.html">Federation</a></li></ol></li><li class="chapter-item expanded "><a href="manhole.html">Manhole</a></li><li class="chapter-item expanded "><a href="metrics-howto.html">Monitoring</a></li><li class="chapter-item expanded "><a href="usage/administration/understanding_synapse_through_grafana_graphs.html">Understanding Synapse Through Grafana Graphs</a></li><li class="chapter-item expanded "><a href="usage/administration/useful_sql_for_admins.html">Useful SQL for Admins</a></li><li class="chapter-item expanded "><a href="usage/administration/database_maintenance_tools.html">Database Maintenance Tools</a></li><li class="chapter-item expanded "><a href="usage/administration/state_groups.html">State Groups</a></li><li class="chapter-item expanded "><a href="usage/administration/request_log.html">Request log format</a></li><li class="chapter-item expanded "><a href="usage/administration/admin_faq.html">Admin FAQ</a></li><li class="chapter-item expanded "><div>Scripts</div></li></ol></li><li class="chapter-item expanded "><li class="part-title">Development</li><li class="chapter-item expanded "><a href="development/contributing_guide.html">Contributing Guide</a></li><li class="chapter-item expanded "><a href="code_style.html">Code Style</a></li><li class="chapter-item expanded "><a href="development/releases.html">Release Cycle</a></li><li class="chapter-item expanded "><a href="development/git.html">Git Usage</a></li><li class="chapter-item expanded "><div>Testing</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/demo.html">Demo scripts</a></li></ol></li><li class="chapter-item expanded "><a href="opentracing.html">OpenTracing</a></li><li class="chapter-item expanded "><a href="development/database_schema.html">Database Schemas</a></li><li class="chapter-item expanded "><a href="development/experimental_features.html">Experimental features</a></li><li class="chapter-item expanded "><div>Synapse Architecture</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/synapse_architecture/cancellation.html">Cancellation</a></li><li class="chapter-item expanded "><a href="log_contexts.html">Log Contexts</a></li><li class="chapter-item expanded "><a href="replication.html">Replication</a></li><li class="chapter-item expanded "><a href="tcp_replication.html">TCP Replication</a></li></ol></li><li class="chapter-item expanded "><a href="development/internal_documentation/index.html">Internal Documentation</a></li><li><ol class="section"><li class="chapter-item expanded "><div>Single Sign-On</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="development/saml.html">SAML</a></li><li class="chapter-item expanded "><a href="development/cas.html">CAS</a></li></ol></li><li class="chapter-item expanded "><a href="development/room-dag-concepts.html">Room DAG concepts</a></li><li class="chapter-item expanded "><div>State Resolution</div></li><li><ol class="section"><li class="chapter-item expanded "><a href="auth_chain_difference_algorithm.html">The Auth Chain Difference Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="media_repository.html">Media Repository</a></li><li class="chapter-item expanded "><a href="room_and_user_statistics.html">Room and User Statistics</a></li></ol></li><li class="chapter-item expanded "><div>Scripts</div></li><li class="chapter-item expanded affix "><li class="part-title">Other</li><li class="chapter-item expanded "><a href="deprecation_policy.html">Dependency Deprecation Policy</a></li><li class="chapter-item expanded "><a href="other/running_synapse_on_single_board_computers.html">Running Synapse on a Single-Board Computer</a></li></ol> </div> <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div> </nav> @@ -150,11 +150,11 @@ team.</p> <h2 id="installing-and-using-synapse"><a class="header" href="#installing-and-using-synapse">Installing and using Synapse</a></h2> <p>This documentation covers topics for <strong>installation</strong>, <strong>configuration</strong> and -<strong>maintainence</strong> of your Synapse process:</p> +<strong>maintenance</strong> of your Synapse process:</p> <ul> <li> <p>Learn how to <a href="setup/installation.html">install</a> and -<a href="usage/configuration/index.html">configure</a> your own instance, perhaps with <a href="usage/configuration/user_authentication/index.html">Single +<a href="usage/configuration/config_documentation.html">configure</a> your own instance, perhaps with <a href="usage/configuration/user_authentication/index.html">Single Sign-On</a>.</p> </li> <li> @@ -220,7 +220,7 @@ do so!</p> <h2 id="donating-to-synapse-development"><a class="header" href="#donating-to-synapse-development">Donating to Synapse development</a></h2> <p>Want to help keep Synapse going but don't know how to code? Synapse is a <a href="https://matrix.org">Matrix.org Foundation</a> project. Consider becoming a -supportor on <a href="https://liberapay.com/matrixdotorg">Liberapay</a>, +supporter on <a href="https://liberapay.com/matrixdotorg">Liberapay</a>, <a href="https://patreon.com/matrixdotorg">Patreon</a> or through <a href="https://paypal.me/matrixdotorg">PayPal</a> via a one-time donation.</p> <p>If you are an organisation or enterprise and would like to sponsor development, @@ -1631,6 +1631,90 @@ dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb </code></pre> </li> </ul> +<h1 id="upgrading-to-v1600"><a class="header" href="#upgrading-to-v1600">Upgrading to v1.60.0</a></h1> +<h2 id="adding-a-new-unique-index-to-state_group_edges-could-fail-if-your-database-is-corrupted"><a class="header" href="#adding-a-new-unique-index-to-state_group_edges-could-fail-if-your-database-is-corrupted">Adding a new unique index to <code>state_group_edges</code> could fail if your database is corrupted</a></h2> +<p>This release of Synapse will add a unique index to the <code>state_group_edges</code> table, in order +to prevent accidentally introducing duplicate information (for example, because a database +backup was restored multiple times).</p> +<p>Duplicate rows being present in this table could cause drastic performance problems; see +<a href="https://github.com/matrix-org/synapse/issues/11779">issue 11779</a> for more details.</p> +<p>If your Synapse database already has had duplicate rows introduced into this table, +this could fail, with either of these errors:</p> +<p><strong>On Postgres:</strong></p> +<pre><code>synapse.storage.background_updates - 623 - INFO - background_updates-0 - Adding index state_group_edges_unique_idx to state_group_edges +synapse.storage.background_updates - 282 - ERROR - background_updates-0 - Error doing update +... +psycopg2.errors.UniqueViolation: could not create unique index "state_group_edges_unique_idx" +DETAIL: Key (state_group, prev_state_group)=(2, 1) is duplicated. +</code></pre> +<p>(The numbers may be different.)</p> +<p><strong>On SQLite:</strong></p> +<pre><code>synapse.storage.background_updates - 623 - INFO - background_updates-0 - Adding index state_group_edges_unique_idx to state_group_edges +synapse.storage.background_updates - 282 - ERROR - background_updates-0 - Error doing update +... +sqlite3.IntegrityError: UNIQUE constraint failed: state_group_edges.state_group, state_group_edges.prev_state_group +</code></pre> +<details> +<summary><b>Expand this section for steps to resolve this problem</b></summary> +<h3 id="on-postgres"><a class="header" href="#on-postgres">On Postgres</a></h3> +<p>Connect to your database with <code>psql</code>.</p> +<pre><code class="language-sql">BEGIN; +DELETE FROM state_group_edges WHERE (ctid, state_group, prev_state_group) IN ( + SELECT row_id, state_group, prev_state_group + FROM ( + SELECT + ctid AS row_id, + MIN(ctid) OVER (PARTITION BY state_group, prev_state_group) AS min_row_id, + state_group, + prev_state_group + FROM state_group_edges + ) AS t1 + WHERE row_id <> min_row_id +); +COMMIT; +</code></pre> +<h3 id="on-sqlite"><a class="header" href="#on-sqlite">On SQLite</a></h3> +<p>At the command-line, use <code>sqlite3 path/to/your-homeserver-database.db</code>:</p> +<pre><code class="language-sql">BEGIN; +DELETE FROM state_group_edges WHERE (rowid, state_group, prev_state_group) IN ( + SELECT row_id, state_group, prev_state_group + FROM ( + SELECT + rowid AS row_id, + MIN(rowid) OVER (PARTITION BY state_group, prev_state_group) AS min_row_id, + state_group, + prev_state_group + FROM state_group_edges + ) + WHERE row_id <> min_row_id +); +COMMIT; +</code></pre> +<h3 id="for-more-details"><a class="header" href="#for-more-details">For more details</a></h3> +<p><a href="https://github.com/matrix-org/synapse/issues/11779#issuecomment-1131545970">This comment on issue 11779</a> +has queries that can be used to check a database for this problem in advance.</p> +</details> +<h2 id="new-signature-for-the-spam-checker-callback-check_event_for_spam"><a class="header" href="#new-signature-for-the-spam-checker-callback-check_event_for_spam">New signature for the spam checker callback <code>check_event_for_spam</code></a></h2> +<p>The previous signature has been deprecated.</p> +<p>Whereas <code>check_event_for_spam</code> callbacks used to return <code>Union[str, bool]</code>, they should now return <code>Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]</code>.</p> +<p>This is part of an ongoing refactoring of the SpamChecker API to make it less ambiguous and more powerful.</p> +<p>If your module implements <code>check_event_for_spam</code> as follows:</p> +<pre><code class="language-python">async def check_event_for_spam(event): + if ...: + # Event is spam + return True + # Event is not spam + return False +</code></pre> +<p>you should rewrite it as follows:</p> +<pre><code class="language-python">async def check_event_for_spam(event): + if ...: + # Event is spam, mark it as forbidden (you may use some more precise error + # code if it is useful). + return synapse.module_api.errors.Codes.FORBIDDEN + # Event is not spam, mark it as such. + return synapse.module_api.NOT_SPAM +</code></pre> <h1 id="upgrading-to-v1590"><a class="header" href="#upgrading-to-v1590">Upgrading to v1.59.0</a></h1> <h2 id="device-name-lookup-over-federation-has-been-disabled-by-default"><a class="header" href="#device-name-lookup-over-federation-has-been-disabled-by-default">Device name lookup over federation has been disabled by default</a></h2> <p>The names of user devices are no longer visible to users on other homeservers by default. @@ -3038,6 +3122,13 @@ followed by a letter. Letters have the following meanings:</p> </ul> <p>For example, setting <code>redaction_retention_period: 5m</code> would remove redacted messages from the database after 5 minutes, rather than 5 months.</p> +<p>In addition, configuration options referring to size use the following suffixes:</p> +<ul> +<li><code>M</code> = MiB, or 1,048,576 bytes</li> +<li><code>K</code> = KiB, or 1024 bytes </li> +</ul> +<p>For example, setting <code>max_avatar_size: 10M</code> means that Synapse will not accept files larger than 10,485,760 bytes +for a user avatar. </p> <h3 id="yaml"><a class="header" href="#yaml">YAML</a></h3> <p>The configuration file is a <a href="https://yaml.org/">YAML</a> file, which means that certain syntax rules apply if you want your config file to be read properly. A few helpful things to know:</p> @@ -3386,7 +3477,7 @@ additional endpoints which should be loaded via dynamic modules.</p> <p>Valid resource names are:</p> <ul> <li> -<p><code>client</code>: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies 'media' and 'static'.</p> +<p><code>client</code>: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies <code>media</code> and <code>static</code>.</p> </li> <li> <p><code>consent</code>: user consent forms (/_matrix/consent). See <a href="usage/configuration/../../consent_tracking.html">here</a> for more.</p> @@ -3395,7 +3486,7 @@ additional endpoints which should be loaded via dynamic modules.</p> <p><code>federation</code>: the server-server API (/_matrix/federation). Also implies <code>media</code>, <code>keys</code>, <code>openid</code></p> </li> <li> -<p><code>keys</code>: the key discovery API (/_matrix/keys).</p> +<p><code>keys</code>: the key discovery API (/_matrix/key).</p> </li> <li> <p><code>media</code>: the media API (/_matrix/media).</p> @@ -3920,15 +4011,48 @@ with intermittent connections, at the cost of higher memory usage. By default, this is zero, which means that sync responses are not cached at all.</p> </li> +<li> +<p><code>cache_autotuning</code> and its sub-options <code>max_cache_memory_usage</code>, <code>target_cache_memory_usage</code>, and +<code>min_cache_ttl</code> work in conjunction with each other to maintain a balance between cache memory +usage and cache entry availability. You must be using <a href="https://github.com/matrix-org/synapse#help-synapse-is-slow-and-eats-all-my-ramcpu">jemalloc</a> +to utilize this option, and all three of the options must be specified for this feature to work. This option +defaults to off, enable it by providing values for the sub-options listed below. Please note that the feature will not work +and may cause unstable behavior (such as excessive emptying of caches or exceptions) if all of the values are not provided. +Please see the <a href="usage/configuration/config_documentation.html#config-conventions">Config Conventions</a> for information on how to specify memory size and cache expiry +durations.</p> +<ul> +<li><code>max_cache_memory_usage</code> sets a ceiling on how much memory the cache can use before caches begin to be continuously evicted. +They will continue to be evicted until the memory usage drops below the <code>target_memory_usage</code>, set in +the setting below, or until the <code>min_cache_ttl</code> is hit. There is no default value for this option.</li> +<li><code>target_memory_usage</code> sets a rough target for the desired memory usage of the caches. There is no default value +for this option.</li> +<li><code>min_cache_ttl</code> sets a limit under which newer cache entries are not evicted and is only applied when +caches are actively being evicted/<code>max_cache_memory_usage</code> has been exceeded. This is to protect hot caches +from being emptied while Synapse is evicting due to memory. There is no default value for this option. </li> +</ul> +</li> </ul> <p>Example configuration:</p> <pre><code class="language-yaml">caches: global_factor: 1.0 per_cache_factors: get_users_who_share_room_with_user: 2.0 - expire_caches: false sync_response_cache_duration: 2m -</code></pre> + cache_autotuning: + max_cache_memory_usage: 1024M + target_cache_memory_usage: 758M + min_cache_ttl: 5m +</code></pre> +<h3 id="reloading-cache-factors"><a class="header" href="#reloading-cache-factors">Reloading cache factors</a></h3> +<p>The cache factors (i.e. <code>caches.global_factor</code> and <code>caches.per_cache_factors</code>) may be reloaded at any time by sending a +<a href="https://en.wikipedia.org/wiki/SIGHUP"><code>SIGHUP</code></a> signal to Synapse using e.g.</p> +<pre><code class="language-commandline">kill -HUP [PID_OF_SYNAPSE_PROCESS] +</code></pre> +<p>If you are running multiple workers, you must individually update the worker +config file and send this signal to each worker process.</p> +<p>If you're using the <a href="https://github.com/matrix-org/synapse/blob/develop/contrib/systemd/matrix-synapse.service">example systemd service</a> +file in Synapse's <code>contrib</code> directory, you can send a <code>SIGHUP</code> signal by using +<code>systemctl reload matrix-synapse</code>.</p> <hr /> <h2 id="database"><a class="header" href="#database">Database</a></h2> <p>Config options related to database settings.</p> @@ -3966,13 +4090,13 @@ connection pool. For a reference to valid arguments, see:</p> <p>For more information on using Synapse with Postgres, see <a href="usage/configuration/../../postgres.html">here</a>.</p> <p>Example SQLite configuration:</p> -<pre><code>database: +<pre><code class="language-yaml">database: name: sqlite3 args: database: /path/to/homeserver.db </code></pre> <p>Example Postgres configuration:</p> -<pre><code>database: +<pre><code class="language-yaml">database: name: psycopg2 txn_limit: 10000 args: @@ -4108,6 +4232,17 @@ Defaults to <code>per_second: 0.003</code>, <code>burst_count: 5</code>.</p> <p>This option sets ratelimiting how often invites can be sent in a room or to a specific user. <code>per_room</code> defaults to <code>per_second: 0.3</code>, <code>burst_count: 10</code> and <code>per_user</code> defaults to <code>per_second: 0.003</code>, <code>burst_count: 5</code>. </p> +<p>Client requests that invite user(s) when <a href="https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom">creating a +room</a> +will count against the <code>rc_invites.per_room</code> limit, whereas +client requests to <a href="https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3roomsroomidinvite">invite a single user to a +room</a> +will count against both the <code>rc_invites.per_user</code> and <code>rc_invites.per_room</code> limits.</p> +<p>Federation requests to invite a user will count against the <code>rc_invites.per_user</code> +limit only, as Synapse presumes ratelimiting by room will be done by the sending server.</p> +<p>The <code>rc_invites.per_user</code> limit applies to the <em>receiver</em> of the invite, rather than the +sender, meaning that a <code>rc_invite.per_user.burst_count</code> of 5 mandates that a single user +cannot <em>receive</em> more than a burst of 5 invites at a time.</p> <p>Example configuration:</p> <pre><code class="language-yaml">rc_invites: per_room: @@ -4362,10 +4497,10 @@ using quality value syntax (;q=). '*' translates to any language.</p> <p>Defaults to "en".</p> <p>Example configuration:</p> <pre><code class="language-yaml"> url_preview_accept_language: - - en-UK - - en-US;q=0.9 - - fr;q=0.8 - - *;q=0.7 + - 'en-UK' + - 'en-US;q=0.9' + - 'fr;q=0.8' + - '*;q=0.7' </code></pre> <hr /> <p>Config option: <code>oembed</code></p> @@ -5414,7 +5549,10 @@ validation will fail without configuring audiences.</li> <p>Use this setting to enable password-based logins. </p> <p>This setting has the following sub-options:</p> <ul> -<li><code>enabled</code>: Defaults to true.</li> +<li><code>enabled</code>: Defaults to true. +Set to false to disable password authentication. +Set to <code>only_for_reauth</code> to allow users with existing passwords to use them +to log in and reauthenticate, whilst preventing new users from setting passwords.</li> <li><code>localdb_enabled</code>: Set to false to disable authentication against the local password database. This is ignored if <code>enabled</code> is false, and is only useful if you have other <code>password_providers</code>. Defaults to true. </li> @@ -5829,6 +5967,25 @@ can publish rooms.</p> action: allow </code></pre> <hr /> +<p>Config option: <code>default_power_level_content_override</code></p> +<p>The <code>default_power_level_content_override</code> option controls the default power +levels for rooms.</p> +<p>Useful if you know that your users need special permissions in rooms +that they create (e.g. to send particular types of state events without +needing an elevated power level). This takes the same shape as the +<code>power_level_content_override</code> parameter in the /createRoom API, but +is applied before that parameter.</p> +<p>Note that each key provided inside a preset (for example <code>events</code> in the example +below) will overwrite all existing defaults inside that key. So in the example +below, newly-created private_chat rooms will have no rules for any event types +except <code>com.example.foo</code>.</p> +<p>Example configuration:</p> +<pre><code class="language-yaml">default_power_level_content_override: + private_chat: { "events": { "com.example.foo" : 0 } } + trusted_private_chat: null + public_chat: null +</code></pre> +<hr /> <h2 id="opentracing"><a class="header" href="#opentracing">Opentracing</a></h2> <p>Configuration options related to Opentracing support.</p> <hr /> @@ -5911,7 +6068,7 @@ specified here must also be in the <code>instance_map</code>.</p> typing: worker1 </code></pre> <hr /> -<p>Config option: <code>run_background_task_on</code></p> +<p>Config option: <code>run_background_tasks_on</code></p> <p>The worker that is used to run background tasks (e.g. cleaning up expired data). If not provided this defaults to the main process.</p> <p>Example configuration:</p> @@ -6268,7 +6425,7 @@ presence: # federation: the server-server API (/_matrix/federation). Also implies # 'media', 'keys', 'openid' # -# keys: the key discovery API (/_matrix/keys). +# keys: the key discovery API (/_matrix/key). # # media: the media API (/_matrix/media). # @@ -6709,6 +6866,12 @@ retention: # A cache 'factor' is a multiplier that can be applied to each of # Synapse's caches in order to increase or decrease the maximum # number of entries that can be stored. +# +# The configuration for cache factors (caches.global_factor and +# caches.per_cache_factors) can be reloaded while the application is running, +# by sending a SIGHUP signal to the Synapse process. Changes to other parts of +# the caching config will NOT be applied after a SIGHUP is received; a restart +# is necessary. # The number of events to cache in memory. Not affected by # caches.global_factor. @@ -6757,6 +6920,24 @@ caches: # #cache_entry_ttl: 30m + # This flag enables cache autotuning, and is further specified by the sub-options `max_cache_memory_usage`, + # `target_cache_memory_usage`, `min_cache_ttl`. These flags work in conjunction with each other to maintain + # a balance between cache memory usage and cache entry availability. You must be using jemalloc to utilize + # this option, and all three of the options must be specified for this feature to work. + #cache_autotuning: + # This flag sets a ceiling on much memory the cache can use before caches begin to be continuously evicted. + # They will continue to be evicted until the memory usage drops below the `target_memory_usage`, set in + # the flag below, or until the `min_cache_ttl` is hit. + #max_cache_memory_usage: 1024M + + # This flag sets a rough target for the desired memory usage of the caches. + #target_cache_memory_usage: 758M + + # 'min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when + # caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches + # from being emptied while Synapse is evicting due to memory. + #min_cache_ttl: 5m + # Controls how long the results of a /sync request are cached for after # a successful response is returned. A higher duration can help clients with # intermittent connections, at the cost of higher memory usage. @@ -8171,7 +8352,9 @@ sso: password_config: - # Uncomment to disable password login + # Uncomment to disable password login. + # Set to `only_for_reauth` to permit reauthentication for users that + # have passwords and are already logged in. # #enabled: false @@ -8441,6 +8624,40 @@ push: # #encryption_enabled_by_default_for_room_type: invite +# Override the default power levels for rooms created on this server, per +# room creation preset. +# +# The appropriate dictionary for the room preset will be applied on top +# of the existing power levels content. +# +# Useful if you know that your users need special permissions in rooms +# that they create (e.g. to send particular types of state events without +# needing an elevated power level). This takes the same shape as the +# `power_level_content_override` parameter in the /createRoom API, but +# is applied before that parameter. +# +# Valid keys are some or all of `private_chat`, `trusted_private_chat` +# and `public_chat`. Inside each of those should be any of the +# properties allowed in `power_level_content_override` in the +# /createRoom API. If any property is missing, its default value will +# continue to be used. If any property is present, it will overwrite +# the existing default completely (so if the `events` property exists, +# the default event power levels will be ignored). +# +#default_power_level_content_override: +# private_chat: +# "events": +# "com.example.myeventtype" : 0 +# "m.room.avatar": 50 +# "m.room.canonical_alias": 50 +# "m.room.encryption": 100 +# "m.room.history_visibility": 100 +# "m.room.name": 50 +# "m.room.power_levels": 100 +# "m.room.server_acl": 100 +# "m.room.tombstone": 100 +# "events_default": 1 + # Uncomment to allow non-server-admin users to create groups on this server # @@ -9374,7 +9591,7 @@ to install Dex.</p> <pre><code class="language-yaml">oidc_providers: - idp_id: keycloak idp_name: "My KeyCloak server" - issuer: "https://127.0.0.1:8443/auth/realms/{realm_name}" + issuer: "https://127.0.0.1:8443/realms/{realm_name}" client_id: "synapse" client_secret: "copy secret generated from above" scopes: ["openid", "profile"] @@ -9502,7 +9719,7 @@ can be used to retrieve information on the authenticated user. As the Synapse login mechanism needs an attribute to uniquely identify users, and that endpoint does not return a <code>sub</code> property, an alternative <code>subject_claim</code> has to be set.</p> <ol> -<li>Create a new OAuth application: https://github.com/settings/applications/new.</li> +<li>Create a new OAuth application: <a href="https://github.com/settings/applications/new">https://github.com/settings/applications/new</a>.</li> <li>Set the callback URL to <code>[synapse public baseurl]/_synapse/client/oidc/callback</code>.</li> </ol> <p>Synapse config:</p> @@ -9527,8 +9744,8 @@ does not return a <code>sub</code> property, an alternative <code>subject_claim< <h3 id="google"><a class="header" href="#google">Google</a></h3> <p><a href="https://developers.google.com/identity/protocols/oauth2/openid-connect">Google</a> is an OpenID certified authentication and authorisation provider.</p> <ol> -<li>Set up a project in the Google API Console (see -https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup).</li> +<li>Set up a project in the Google API Console (see +<a href="https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup">documentation</a>).</li> <li>Add an "OAuth Client ID" for a Web Application under "Credentials".</li> <li>Copy the Client ID and Client Secret, and add the following to your synapse config: <pre><code class="language-yaml">oidc_providers: @@ -9687,8 +9904,8 @@ private key with "SiWA" enabled.</p> <li>Team ID: a 10-character ID associated with your developer account.</li> <li>Key ID: the 10-character identifier for the key.</li> </ul> -<p>https://help.apple.com/developer-account/?lang=en#/dev77c875b7e has more -documentation on setting up SiWA.</p> +<p><a href="https://help.apple.com/developer-account/?lang=en#/dev77c875b7e">Apple's developer documentation</a> +has more information on setting up SiWA.</p> <p>The synapse config will look like this:</p> <pre><code class="language-yaml"> - idp_id: apple idp_name: Apple @@ -9715,10 +9932,10 @@ needed to add OAuth2 capabilities to your Django projects. It supports <a href="https://django-oauth-toolkit.readthedocs.io/en/latest/oidc.html">OpenID Connect too</a>.</p> <p>Configuration on Django's side:</p> <ol> -<li>Add an application: https://example.com/admin/oauth2_provider/application/add/ and choose parameters like this:</li> +<li>Add an application: <code>https://example.com/admin/oauth2_provider/application/add/</code> and choose parameters like this:</li> </ol> <ul> -<li><code>Redirect uris</code>: https://synapse.example.com/_synapse/client/oidc/callback</li> +<li><code>Redirect uris</code>: <code>https://synapse.example.com/_synapse/client/oidc/callback</code></li> <li><code>Client type</code>: <code>Confidential</code></li> <li><code>Authorization grant type</code>: <code>Authorization code</code></li> <li><code>Algorithm</code>: <code>HMAC with SHA-2 256</code></li> @@ -11013,20 +11230,25 @@ Synapse instances. Spam checker callbacks can be registered using the module API <p>The available spam checker callbacks are:</p> <h3 id="check_event_for_spam"><a class="header" href="#check_event_for_spam"><code>check_event_for_spam</code></a></h3> <p><em>First introduced in Synapse v1.37.0</em></p> -<pre><code class="language-python">async def check_event_for_spam(event: "synapse.events.EventBase") -> Union[bool, str] +<p><em>Changed in Synapse v1.60.0: <code>synapse.module_api.NOT_SPAM</code> and <code>synapse.module_api.errors.Codes</code> can be returned by this callback. Returning a boolean or a string is now deprecated.</em> </p> +<pre><code class="language-python">async def check_event_for_spam(event: "synapse.module_api.EventBase") -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", str, bool] </code></pre> -<p>Called when receiving an event from a client or via federation. The callback must return -either:</p> +<p>Called when receiving an event from a client or via federation. The callback must return one of:</p> <ul> -<li>an error message string, to indicate the event must be rejected because of spam and -give a rejection reason to forward to clients;</li> -<li>the boolean <code>True</code>, to indicate that the event is spammy, but not provide further details; or</li> -<li>the booelan <code>False</code>, to indicate that the event is not considered spammy.</li> +<li><code>synapse.module_api.NOT_SPAM</code>, to allow the operation. Other callbacks may still +decide to reject it.</li> +<li><code>synapse.module_api.errors.Codes</code> to reject the operation with an error code. In case +of doubt, <code>synapse.module_api.errors.Codes.FORBIDDEN</code> is a good error code.</li> +<li>(deprecated) a non-<code>Codes</code> <code>str</code> to reject the operation and specify an error message. Note that clients +typically will not localize the error message to the user's preferred locale.</li> +<li>(deprecated) <code>False</code>, which is the same as returning <code>synapse.module_api.NOT_SPAM</code>.</li> +<li>(deprecated) <code>True</code>, which is the same as returning <code>synapse.module_api.errors.Codes.FORBIDDEN</code>.</li> </ul> <p>If multiple modules implement this callback, they will be considered in order. If a -callback returns <code>False</code>, Synapse falls through to the next one. The value of the first -callback that does not return <code>False</code> will be used. If this happens, Synapse will not call -any of the subsequent implementations of this callback.</p> +callback returns <code>synapse.module_api.NOT_SPAM</code>, Synapse falls through to the next one. +The value of the first callback that does not return <code>synapse.module_api.NOT_SPAM</code> will +be used. If this happens, Synapse will not call any of the subsequent implementations of +this callback.</p> <h3 id="user_may_join_room"><a class="header" href="#user_may_join_room"><code>user_may_join_room</code></a></h3> <p><em>First introduced in Synapse v1.37.0</em></p> <pre><code class="language-python">async def user_may_join_room(user: str, room: str, is_invited: bool) -> bool @@ -11186,6 +11408,18 @@ whether the given file should be excluded from the homeserver's media store. Ret callback returns <code>False</code>, Synapse falls through to the next one. The value of the first callback that does not return <code>False</code> will be used. If this happens, Synapse will not call any of the subsequent implementations of this callback.</p> +<h3 id="should_drop_federated_event"><a class="header" href="#should_drop_federated_event"><code>should_drop_federated_event</code></a></h3> +<p><em>First introduced in Synapse v1.60.0</em></p> +<pre><code class="language-python">async def should_drop_federated_event(event: "synapse.events.EventBase") -> bool +</code></pre> +<p>Called when checking whether a remote server can federate an event with us. <strong>Returning +<code>True</code> from this function will silently drop a federated event and split-brain our view +of a room's DAG, and thus you shouldn't use this callback unless you know what you are +doing.</strong></p> +<p>If multiple modules implement this callback, they will be considered in order. If a +callback returns <code>False</code>, Synapse falls through to the next one. The value of the first +callback that does not return <code>False</code> will be used. If this happens, Synapse will not call +any of the subsequent implementations of this callback.</p> <h2 id="example"><a class="header" href="#example">Example</a></h2> <p>The example below is a module that implements the spam checker callback <code>check_event_for_spam</code> to deny any message sent by users whose Matrix user IDs are @@ -12168,6 +12402,9 @@ information.</p> # Presence requests ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ + +# User directory search requests +^/_matrix/client/(r0|v3|unstable)/user_directory/search$ </code></pre> <p>Additionally, the following REST endpoints can be handled for GET requests:</p> <pre><code>^/_matrix/federation/v1/groups/ @@ -12343,6 +12580,12 @@ worker_log_config: /etc/matrix-synapse/background-worker-log.yaml </code></pre> <p>This work cannot be load-balanced; please ensure the main process is restarted after setting this option in the shared configuration!</p> +<p>User directory updates allow REST endpoints matching the following regular +expressions to work:</p> +<pre><code>^/_matrix/client/(r0|v3|unstable)/user_directory/search$ +</code></pre> +<p>The above endpoints can be routed to any worker, though you may choose to route +it to the chosen user directory worker.</p> <p>This style of configuration supersedes the legacy <code>synapse.app.user_dir</code> worker application type.</p> <h4 id="notifying-application-services"><a class="header" href="#notifying-application-services">Notifying Application Services</a></h4> @@ -13100,7 +13343,7 @@ If <code>false</code> these files will be deleted. Defaults to <code>true</code> </code></pre> <p>URL Parameters</p> <ul> -<li><code>unix_timestamp_in_ms</code>: string representing a positive integer - Unix timestamp in milliseconds. +<li><code>before_ts</code>: string representing a positive integer - Unix timestamp in milliseconds. All cached media that was last accessed before this timestamp will be removed.</li> </ul> <p>Response:</p> @@ -16351,7 +16594,20 @@ in production (and indeed, we have some code paths specific to each database). This means that we need to run our unit tests against PostgreSQL too. Our CI does this automatically for pull requests and release candidates, but it's sometimes useful to reproduce this locally.</p> -<p>To do so, <a href="development/../postgres.html">configure Postgres</a> and run <code>trial</code> with the +<h4 id="using-docker"><a class="header" href="#using-docker">Using Docker</a></h4> +<p>The easiest way to do so is to run Postgres via a docker container. In one +terminal:</p> +<pre><code class="language-shell">docker run --rm -e POSTGRES_PASSWORD=mysecretpassword -e POSTGRES_USER=postgres -e POSTGRES_DB=postgress -p 5432:5432 postgres:14 +</code></pre> +<p>If you see an error like</p> +<pre><code>docker: Error response from daemon: driver failed programming external connectivity on endpoint nice_ride (b57bbe2e251b70015518d00c9981e8cb8346b5c785250341a6c53e3c899875f1): Error starting userland proxy: listen tcp4 0.0.0.0:5432: bind: address already in use. +</code></pre> +<p>then something is already bound to port 5432. You're probably already running postgres locally.</p> +<p>Once you have a postgres server running, invoke <code>trial</code> in a second terminal:</p> +<pre><code class="language-shell">SYNAPSE_POSTGRES=1 SYNAPSE_POSTGRES_HOST=127.0.0.1 SYNAPSE_POSTGRES_USER=postgres SYNAPSE_POSTGRES_PASSWORD=mysecretpassword poetry run trial tests +</code></pre> +<h4 id="using-an-existing-postgres-installation"><a class="header" href="#using-an-existing-postgres-installation">Using an existing Postgres installation</a></h4> +<p>If you have postgres already installed on your system, you can run <code>trial</code> with the following environment variables matching your configuration:</p> <ul> <li><code>SYNAPSE_POSTGRES</code> to anything nonempty</li> @@ -16370,7 +16626,8 @@ trial <p>You don't need to specify the host, user, port or password if your Postgres server is set to authenticate you over the UNIX socket (i.e. if the <code>psql</code> command works without further arguments).</p> -<p>Your Postgres account needs to be able to create databases.</p> +<p>Your Postgres account needs to be able to create databases; see the postgres +docs for <a href="https://www.postgresql.org/docs/current/sql-alterrole.html"><code>ALTER ROLE</code></a>.</p> <h2 id="run-the-integration-tests-sytest"><a class="header" href="#run-the-integration-tests-sytest">Run the integration tests (<a href="https://github.com/matrix-org/sytest">Sytest</a>).</a></h2> <p>The integration tests are a more comprehensive suite of tests. They run a full version of Synapse, including your changes, to check if @@ -16496,8 +16753,8 @@ and you agree to license it under the same terms as the project's license, we've same lightweight approach that the Linux Kernel <a href="https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin%3E">submitting patches process</a>, <a href="https://github.com/docker/docker/blob/master/CONTRIBUTING.md">Docker</a>, and many other -projects use: the DCO (Developer Certificate of Origin: -http://developercertificate.org/). This is a simple declaration that you wrote +projects use: the DCO (<a href="http://developercertificate.org/">Developer Certificate of Origin</a>). +This is a simple declaration that you wrote the contribution or otherwise have the right to contribute it to Matrix:</p> <pre><code>Developer Certificate of Origin Version 1.1 @@ -16903,7 +17160,7 @@ the history of Synapse), we've had two releases in flight at once. Obviously, <p><strong>DO NOT USE THESE DEMO SERVERS IN PRODUCTION</strong></p> <p>Requires you to have a <a href="https://matrix-org.github.io/synapse/develop/development/contributing_guide.html#4-install-the-dependencies">Synapse development environment setup</a>.</p> <p>The demo setup allows running three federation Synapse servers, with server -names <code>localhost:8080</code>, <code>localhost:8081</code>, and <code>localhost:8082</code>.</p> +names <code>localhost:8480</code>, <code>localhost:8481</code>, and <code>localhost:8482</code>.</p> <p>You can access them via any Matrix client over HTTP at <code>localhost:8080</code>, <code>localhost:8081</code>, and <code>localhost:8082</code> or over HTTPS at <code>localhost:8480</code>, <code>localhost:8481</code>, and <code>localhost:8482</code>.</p> @@ -16915,9 +17172,10 @@ and the servers are configured in a highly insecure way, including:</p> </ul> <p>The servers are configured to store their data under <code>demo/8080</code>, <code>demo/8081</code>, and <code>demo/8082</code>. This includes configuration, logs, SQLite databases, and media.</p> -<p>Note that when joining a public room on a different HS via "#foo:bar.net", then -you are (in the current impl) joining a room with room_id "foo". This means that -it won't work if your HS already has a room with that name.</p> +<p>Note that when joining a public room on a different homeserver via "#foo:bar.net", +then you are (in the current implementation) joining a room with room_id "foo". +This means that it won't work if your homeserver already has a room with that +name.</p> <h2 id="using-the-demo-scripts"><a class="header" href="#using-the-demo-scripts">Using the demo scripts</a></h2> <p>There's three main scripts with straightforward purposes:</p> <ul> @@ -17201,6 +17459,353 @@ but one should be used if unsure.</p> <p>New experimental configuration flags should be added under the <code>experimental</code> configuration key (see the <code>synapse.config.experimental</code> file) and either explain (briefly) what is being enabled, or include the MSC number.</p> +<div style="break-before: page; page-break-before: always;"></div><h1 id="cancellation"><a class="header" href="#cancellation">Cancellation</a></h1> +<p>Sometimes, requests take a long time to service and clients disconnect +before Synapse produces a response. To avoid wasting resources, Synapse +can cancel request processing for select endpoints marked with the +<code>@cancellable</code> decorator.</p> +<p>Synapse makes use of Twisted's <code>Deferred.cancel()</code> feature to make +cancellation work. The <code>@cancellable</code> decorator does nothing by itself +and merely acts as a flag, signalling to developers and other code alike +that a method can be cancelled.</p> +<h2 id="enabling-cancellation-for-an-endpoint"><a class="header" href="#enabling-cancellation-for-an-endpoint">Enabling cancellation for an endpoint</a></h2> +<ol> +<li>Check that the endpoint method, and any <code>async</code> functions in its call +tree handle cancellation correctly. See +<a href="development/synapse_architecture/cancellation.html#handling-cancellation-correctly">Handling cancellation correctly</a> +for a list of things to look out for.</li> +<li>Add the <code>@cancellable</code> decorator to the <code>on_GET/POST/PUT/DELETE</code> +method. It's not recommended to make non-<code>GET</code> methods cancellable, +since cancellation midway through some database updates is less +likely to be handled correctly.</li> +</ol> +<h2 id="mechanics"><a class="header" href="#mechanics">Mechanics</a></h2> +<p>There are two stages to cancellation: downward propagation of a +<code>cancel()</code> call, followed by upwards propagation of a <code>CancelledError</code> +out of a blocked <code>await</code>. +Both Twisted and asyncio have a cancellation mechanism.</p> +<table><thead><tr><th></th><th>Method</th><th>Exception</th><th>Exception inherits from</th></tr></thead><tbody> +<tr><td>Twisted</td><td><code>Deferred.cancel()</code></td><td><code>twisted.internet.defer.CancelledError</code></td><td><code>Exception</code> (!)</td></tr> +<tr><td>asyncio</td><td><code>Task.cancel()</code></td><td><code>asyncio.CancelledError</code></td><td><code>BaseException</code></td></tr> +</tbody></table> +<h3 id="deferredcancel"><a class="header" href="#deferredcancel">Deferred.cancel()</a></h3> +<p>When Synapse starts handling a request, it runs the async method +responsible for handling it using <code>defer.ensureDeferred</code>, which returns +a <code>Deferred</code>. For example:</p> +<pre><code class="language-python">def do_something() -> Deferred[None]: + ... + +@cancellable +async def on_GET() -> Tuple[int, JsonDict]: + d = make_deferred_yieldable(do_something()) + await d + return 200, {} + +request = defer.ensureDeferred(on_GET()) +</code></pre> +<p>When a client disconnects early, Synapse checks for the presence of the +<code>@cancellable</code> decorator on <code>on_GET</code>. Since <code>on_GET</code> is cancellable, +<code>Deferred.cancel()</code> is called on the <code>Deferred</code> from +<code>defer.ensureDeferred</code>, ie. <code>request</code>. Twisted knows which <code>Deferred</code> +<code>request</code> is waiting on and passes the <code>cancel()</code> call on to <code>d</code>.</p> +<p>The <code>Deferred</code> being waited on, <code>d</code>, may have its own handling for +<code>cancel()</code> and pass the call on to other <code>Deferred</code>s.</p> +<p>Eventually, a <code>Deferred</code> handles the <code>cancel()</code> call by resolving itself +with a <code>CancelledError</code>.</p> +<h3 id="cancellederror"><a class="header" href="#cancellederror">CancelledError</a></h3> +<p>The <code>CancelledError</code> gets raised out of the <code>await</code> and bubbles up, as +per normal Python exception handling.</p> +<h2 id="handling-cancellation-correctly"><a class="header" href="#handling-cancellation-correctly">Handling cancellation correctly</a></h2> +<p>In general, when writing code that might be subject to cancellation, two +things must be considered:</p> +<ul> +<li>The effect of <code>CancelledError</code>s raised out of <code>await</code>s.</li> +<li>The effect of <code>Deferred</code>s being <code>cancel()</code>ed.</li> +</ul> +<p>Examples of code that handles cancellation incorrectly include:</p> +<ul> +<li><code>try-except</code> blocks which swallow <code>CancelledError</code>s.</li> +<li>Code that shares the same <code>Deferred</code>, which may be cancelled, between +multiple requests.</li> +<li>Code that starts some processing that's exempt from cancellation, but +uses a logging context from cancellable code. The logging context +will be finished upon cancellation, while the uncancelled processing +is still using it.</li> +</ul> +<p>Some common patterns are listed below in more detail.</p> +<h3 id="async-function-calls"><a class="header" href="#async-function-calls"><code>async</code> function calls</a></h3> +<p>Most functions in Synapse are relatively straightforward from a +cancellation standpoint: they don't do anything with <code>Deferred</code>s and +purely call and <code>await</code> other <code>async</code> functions.</p> +<p>An <code>async</code> function handles cancellation correctly if its own code +handles cancellation correctly and all the async function it calls +handle cancellation correctly. For example:</p> +<pre><code class="language-python">async def do_two_things() -> None: + check_something() + await do_something() + await do_something_else() +</code></pre> +<p><code>do_two_things</code> handles cancellation correctly if <code>do_something</code> and +<code>do_something_else</code> handle cancellation correctly.</p> +<p>That is, when checking whether a function handles cancellation +correctly, its implementation and all its <code>async</code> function calls need to +be checked, recursively.</p> +<p>As <code>check_something</code> is not <code>async</code>, it does not need to be checked.</p> +<h3 id="cancellederrors"><a class="header" href="#cancellederrors">CancelledErrors</a></h3> +<p>Because Twisted's <code>CancelledError</code>s are <code>Exception</code>s, it's easy to +accidentally catch and suppress them. Care must be taken to ensure that +<code>CancelledError</code>s are allowed to propagate upwards.</p> +<table width="100%"> +<tr> +<td width="50%" valign="top"> +<p><strong>Bad</strong>:</p> +<pre><code class="language-python">try: + await do_something() +except Exception: + # `CancelledError` gets swallowed here. + logger.info(...) +</code></pre> +</td> +<td width="50%" valign="top"> +<p><strong>Good</strong>:</p> +<pre><code class="language-python">try: + await do_something() +except CancelledError: + raise +except Exception: + logger.info(...) +</code></pre> +</td> +</tr> +<tr> +<td width="50%" valign="top"> +<p><strong>OK</strong>:</p> +<pre><code class="language-python">try: + check_something() + # A `CancelledError` won't ever be raised here. +except Exception: + logger.info(...) +</code></pre> +</td> +<td width="50%" valign="top"> +<p><strong>Good</strong>:</p> +<pre><code class="language-python">try: + await do_something() +except ValueError: + logger.info(...) +</code></pre> +</td> +</tr> +</table> +<h4 id="defergatherresults"><a class="header" href="#defergatherresults">defer.gatherResults</a></h4> +<p><code>defer.gatherResults</code> produces a <code>Deferred</code> which:</p> +<ul> +<li>broadcasts <code>cancel()</code> calls to every <code>Deferred</code> being waited on.</li> +<li>wraps the first exception it sees in a <code>FirstError</code>.</li> +</ul> +<p>Together, this means that <code>CancelledError</code>s will be wrapped in +a <code>FirstError</code> unless unwrapped. Such <code>FirstError</code>s are liable to be +swallowed, so they must be unwrapped.</p> +<table width="100%"> +<tr> +<td width="50%" valign="top"> +<p><strong>Bad</strong>:</p> +<pre><code class="language-python">async def do_something() -> None: + await make_deferred_yieldable( + defer.gatherResults([...], consumeErrors=True) + ) + +try: + await do_something() +except CancelledError: + raise +except Exception: + # `FirstError(CancelledError)` gets swallowed here. + logger.info(...) +</code></pre> +</td> +<td width="50%" valign="top"> +<p><strong>Good</strong>:</p> +<pre><code class="language-python">async def do_something() -> None: + await make_deferred_yieldable( + defer.gatherResults([...], consumeErrors=True) + ).addErrback(unwrapFirstError) + +try: + await do_something() +except CancelledError: + raise +except Exception: + logger.info(...) +</code></pre> +</td> +</tr> +</table> +<h3 id="creation-of-deferreds"><a class="header" href="#creation-of-deferreds">Creation of <code>Deferred</code>s</a></h3> +<p>If a function creates a <code>Deferred</code>, the effect of cancelling it must be considered. <code>Deferred</code>s that get shared are likely to have unintended behaviour when cancelled.</p> +<table width="100%"> +<tr> +<td width="50%" valign="top"> +<p><strong>Bad</strong>:</p> +<pre><code class="language-python">cache: Dict[str, Deferred[None]] = {} + +def wait_for_room(room_id: str) -> Deferred[None]: + deferred = cache.get(room_id) + if deferred is None: + deferred = Deferred() + cache[room_id] = deferred + # `deferred` can have multiple waiters. + # All of them will observe a `CancelledError` + # if any one of them is cancelled. + return make_deferred_yieldable(deferred) + +# Request 1 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +# Request 2 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +</code></pre> +</td> +<td width="50%" valign="top"> +<p><strong>Good</strong>:</p> +<pre><code class="language-python">cache: Dict[str, Deferred[None]] = {} + +def wait_for_room(room_id: str) -> Deferred[None]: + deferred = cache.get(room_id) + if deferred is None: + deferred = Deferred() + cache[room_id] = deferred + # `deferred` will never be cancelled now. + # A `CancelledError` will still come out of + # the `await`. + # `delay_cancellation` may also be used. + return make_deferred_yieldable(stop_cancellation(deferred)) + +# Request 1 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +# Request 2 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +</code></pre> +</td> +</tr> +<tr> +<td width="50%" valign="top"> +</td> +<td width="50%" valign="top"> +<p><strong>Good</strong>:</p> +<pre><code class="language-python">cache: Dict[str, List[Deferred[None]]] = {} + +def wait_for_room(room_id: str) -> Deferred[None]: + if room_id not in cache: + cache[room_id] = [] + # Each request gets its own `Deferred` to wait on. + deferred = Deferred() + cache[room_id]].append(deferred) + return make_deferred_yieldable(deferred) + +# Request 1 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +# Request 2 +await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org") +</code></pre> +</td> +</table> +<h3 id="uncancelled-processing"><a class="header" href="#uncancelled-processing">Uncancelled processing</a></h3> +<p>Some <code>async</code> functions may kick off some <code>async</code> processing which is +intentionally protected from cancellation, by <code>stop_cancellation</code> or +other means. If the <code>async</code> processing inherits the logcontext of the +request which initiated it, care must be taken to ensure that the +logcontext is not finished before the <code>async</code> processing completes.</p> +<table width="100%"> +<tr> +<td width="50%" valign="top"> +<p><strong>Bad</strong>:</p> +<pre><code class="language-python">cache: Optional[ObservableDeferred[None]] = None + +async def do_something_else( + to_resolve: Deferred[None] +) -> None: + await ... + logger.info("done!") + to_resolve.callback(None) + +async def do_something() -> None: + if not cache: + to_resolve = Deferred() + cache = ObservableDeferred(to_resolve) + # `do_something_else` will never be cancelled and + # can outlive the `request-1` logging context. + run_in_background(do_something_else, to_resolve) + + await make_deferred_yieldable(cache.observe()) + +with LoggingContext("request-1"): + await do_something() +</code></pre> +</td> +<td width="50%" valign="top"> +<p><strong>Good</strong>:</p> +<pre><code class="language-python">cache: Optional[ObservableDeferred[None]] = None + +async def do_something_else( + to_resolve: Deferred[None] +) -> None: + await ... + logger.info("done!") + to_resolve.callback(None) + +async def do_something() -> None: + if not cache: + to_resolve = Deferred() + cache = ObservableDeferred(to_resolve) + run_in_background(do_something_else, to_resolve) + # We'll wait until `do_something_else` is + # done before raising a `CancelledError`. + await make_deferred_yieldable( + delay_cancellation(cache.observe()) + ) + else: + await make_deferred_yieldable(cache.observe()) + +with LoggingContext("request-1"): + await do_something() +</code></pre> +</td> +</tr> +<tr> +<td width="50%"> +<p><strong>OK</strong>:</p> +<pre><code class="language-python">cache: Optional[ObservableDeferred[None]] = None + +async def do_something_else( + to_resolve: Deferred[None] +) -> None: + await ... + logger.info("done!") + to_resolve.callback(None) + +async def do_something() -> None: + if not cache: + to_resolve = Deferred() + cache = ObservableDeferred(to_resolve) + # `do_something_else` will get its own independent + # logging context. `request-1` will not count any + # metrics from `do_something_else`. + run_as_background_process( + "do_something_else", + do_something_else, + to_resolve, + ) + + await make_deferred_yieldable(cache.observe()) + +with LoggingContext("request-1"): + await do_something() +</code></pre> +</td> +<td width="50%"> +</td> +</tr> +</table> <div style="break-before: page; page-break-before: always;"></div><h1 id="log-contexts"><a class="header" href="#log-contexts">Log Contexts</a></h1> <p>To help track the processing of individual requests, synapse uses a '<code>log context</code>' to track which request it is handling at any given |